diff --git a/.circleci/config.yml b/.circleci/config.yml index 73fb259e2bc3..2a04298eebf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -242,6 +242,7 @@ commands: libgnutls30 \ libh2o-evloop0.13 \ liblmdb0 \ + libnghttp2-dev \ libre2-5 \ libssl-dev \ libsystemd0 \ @@ -250,19 +251,6 @@ commands: protobuf-compiler \ python3-venv - install-ixfrdist-deps: - description: "Install all libraries needed for testing ixfrdist" - steps: - - run: apt-get update - - run: - command: | - apt-get install -qq -y \ - git \ - libboost-all-dev \ - libsystemd0 \ - libyaml-cpp0.6 \ - python3-venv - install-auth-dev-deps: description: Install all packages needed to build the auth steps: @@ -313,7 +301,7 @@ commands: parameters: version: type: string - default: "42" + default: "45" distro: type: string default: "debian" @@ -328,9 +316,9 @@ commands: name: "Install apt-keys" command: | if [ "<< parameters.version >>" = "master" ]; then - curl https://repo.powerdns.com/CBC8B383-pub.asc | apt-key add - + curl https://repo.powerdns.com/CBC8B383-pub.asc -o /etc/apt/trusted.gpg.d/CBC8B383-pub.asc else - curl https://repo.powerdns.com/FD380FBB-pub.asc | apt-key add - + curl https://repo.powerdns.com/FD380FBB-pub.asc -o /etc/apt/trusted.gpg.d/FD380FBB-pub.asc fi - run: name: "Add repository" @@ -596,7 +584,7 @@ jobs: CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ ./configure \ --enable-option-checking=fatal \ - --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe random remote tinydns' \ + --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \ --enable-systemd \ --enable-tools \ --enable-unit-tests \ @@ -726,30 +714,6 @@ jobs: context: godbc_mssql-nsec3-narrow skip: 8bit-txt-unescaped - test-auth-regress-gsqlite3: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - auth-regress: - context: gsqlite3-nodnssec-both - - auth-regress: - context: gsqlite3-both - - auth-regress: - context: gsqlite3-nsec3-both - - auth-regress: - context: gsqlite3-nsec3-optout-both - - auth-regress: - context: gsqlite3-nsec3-narrow - test-auth-regress-bind: resource_class: small @@ -771,16 +735,6 @@ jobs: - auth-regress-setup - run: command: apt-get install -qq -y p11-kit softhsm2 - - auth-regress: - context: bind-both - - auth-regress: - context: bind-dnssec-both - - auth-regress: - context: bind-dnssec-nsec3-both - - auth-regress: - context: bind-dnssec-nsec3-optout-both - - auth-regress: - context: bind-dnssec-nsec3-narrow - auth-regress: context: bind-dnssec-pkcs11 - run: @@ -798,94 +752,6 @@ jobs: - auth-regress: context: bind-hybrid-nsec3 - test-auth-regress-gmysql: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - - image: mysql:5 - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: 1 - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - steps: - - auth-regress-setup - - run: - command: apt-get install -qq -y default-mysql-client - - run: - command: | - cat >> ~/.my.cnf \<<- __EOF__ - [client] - protocol=TCP - default-character-set=latin1 - __EOF__ - - run: - name: Set up mysql client lib to force TCP - command: echo 'export GMYSQLHOST=127.0.0.1 export GMYSQL2HOST=127.0.0.1 ' > ./vars - workdir: ~/project/regression-tests - - auth-regress: - context: gmysql-nodnssec-both - - auth-regress: - context: gmysql-both - - auth-regress: - context: gmysql-nsec3-both - - auth-regress: - context: gmysql-nsec3-optout-both - - auth-regress: - context: gmysql-nsec3-narrow - - auth-regress: - context: gmysql_sp-both - - test-auth-regress-gpgsql: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - - image: postgres:9 - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - POSTGRES_USER: root - POSTGRES_HOST_AUTH_METHOD: trust - steps: - - auth-regress-setup - - run: - command: apt-get install -qq -y postgresql-client - - run: - name: Use TCP for Postgresql connections - command: | - cat >> ./vars \<<- __EOF__ - export PGHOST=127.0.0.1 - export PGPORT=5432 - __EOF__ - workdir: ~/project/regression-tests - - auth-regress: - context: gpgsql-nodnssec-both - - auth-regress: - context: gpgsql-both - - auth-regress: - context: gpgsql-nsec3-both - - auth-regress: - context: gpgsql-nsec3-optout-both - - auth-regress: - context: gpgsql-nsec3-narrow - - auth-regress: - context: gpgsql_sp-both - test-auth-regress-ldap: resource_class: small @@ -919,23 +785,6 @@ jobs: context: ldap-strict doroot: false - test-auth-regress-tinydns: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - auth-regress: - context: tinydns - doroot: false - test-auth-regress-geoip: resource_class: small @@ -949,107 +798,11 @@ jobs: ASAN_OPTIONS: detect_leaks=0 steps: - auth-regress-setup - - auth-regress: - context: geoip - doroot: false - - auth-regress: - context: geoip-nsec3-narrow - doroot: false - run: export geoipdatabase=../modules/geoipbackend/regression-tests/GeoLiteCity.mmdb - auth-regress: context: geoip doroot: false - test-auth-regress-lua2: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - run: - name: Make empty trustedkeys file to avoid useless test script failure - workdir: ~/project/regression-tests - command: touch trustedkeys - - auth-regress: - context: lua2 - doroot: false - - auth-regress: - context: lua2-dnssec - doroot: false - - test-auth-regress-lmdb: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - run: apt-get install -y -qq jq - - auth-regress: - context: lmdb-nodnssec-both - - auth-regress: - context: lmdb-both - - auth-regress: - context: lmdb-nsec3-both - - auth-regress: - context: lmdb-nsec3-optout-both - - auth-regress: - context: lmdb-nsec3-narrow - - test-auth-algorithms: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - run: - name: Test all algorithms - command: /opt/pdns-auth/bin/pdnsutil test-algorithms - - test-auth-py: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - auth-regress-setup - - run: apt-get -y -qq install python3-venv - - run: - name: Run auth-py tests - workdir: ~/project/regression-tests.auth-py - command: | - PDNS="/opt/pdns-auth/sbin/pdns_server" \ - PDNSUTIL="/opt/pdns-auth/bin/pdnsutil" \ - SDIG="/opt/pdns-auth/bin/sdig" \ - ZONE2SQL="/opt/pdns-auth/bin/zone2sql" \ - ./runtests - build-recursor: docker: - image: debian:buster @@ -1147,7 +900,7 @@ jobs: ASAN_OPTIONS: detect_leaks=0 steps: - add-auth-repo - - run: apt-get --no-install-recommends install -qq -y pdns-server pdns-backend-bind pdns-tools daemontools authbind jq libfaketime lua-posix lua-socket moreutils bc python3-venv protobuf-compiler + - run: apt-get --no-install-recommends install -qq -y pdns-server pdns-backend-bind pdns-tools daemontools authbind jq libfaketime lua-posix lua-socket moreutils bc python3-venv protobuf-compiler prometheus - install-recursor-deps - run: name: Set up authbind @@ -1233,156 +986,6 @@ jobs: TRACE=no \ ./timestamp ./recursor-test 5300 50000 16 workdir: ~/project/regression-tests - - run: - name: Run bulktest E - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 2 4096 - workdir: ~/project/regression-tests - - run: - name: Run bulktest F - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 4 4096 - workdir: ~/project/regression-tests - - run: - name: Run bulktest G - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 - workdir: ~/project/regression-tests - - run: - name: Run bulktest H - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 16 4096 - workdir: ~/project/regression-tests - - run: - name: Run bulktest I - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 1 - workdir: ~/project/regression-tests - - run: - name: Run bulktest J - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 2 - workdir: ~/project/regression-tests - - run: - name: Run bulktest K - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 4 - workdir: ~/project/regression-tests - - run: - name: Run bulktest L - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 8 - workdir: ~/project/regression-tests - - run: - name: Run bulktest M - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 16 - workdir: ~/project/regression-tests - - run: - name: Run bulktest N - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 32 - workdir: ~/project/regression-tests - - run: - name: Run bulktest O - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 64 - workdir: ~/project/regression-tests - - run: - name: Run bulktest P - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 128 - workdir: ~/project/regression-tests - - run: - name: Run bulktest Q - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 256 - workdir: ~/project/regression-tests - - run: - name: Run bulktest R - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 512 - workdir: ~/project/regression-tests - - run: - name: Run bulktest S - command: | - DNSBULKTEST=/usr/bin/dnsbulktest \ - RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor \ - RECCONTROL=/opt/pdns-recursor/bin/rec_control \ - THRESHOLD=95 \ - TRACE=no \ - ./timestamp ./recursor-test 5300 50000 8 4096 1024 - workdir: ~/project/regression-tests build-auth-docs: resource_class: small @@ -1491,7 +1094,7 @@ jobs: ./configure \ --enable-option-checking=fatal \ --disable-systemd \ - --with-modules='bind lmdb ldap gmysql gsqlite3 gpgsql godbc random tinydns' \ + --with-modules='bind lmdb ldap gmysql gsqlite3 gpgsql godbc tinydns' \ --enable-tools \ --with-lmdb=/usr \ --with-libsodium \ @@ -1540,9 +1143,10 @@ jobs: libcdb-dev \ libedit-dev \ libfstrm-dev \ - libgnutls30 \ + libgnutls28-dev \ liblmdb-dev \ libluajit-5.1-dev \ + libnghttp2-dev \ libre2-dev \ libsnmp-dev \ libsodium-dev \ @@ -1576,6 +1180,7 @@ jobs: --with-libsodium \ --with-lua=luajit \ --with-libcap \ + --with-nghttp2 \ --with-re2 working_directory: /opt/project/pdns/dnsdistdist - run: @@ -1670,234 +1275,6 @@ jobs: --form description="master build" \ https://scan.coverity.com/builds?project=${COVERITY_PROJECT} - build-dnsdist: - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - run: - name: Install dependencies - command: | - apt-get update && apt-get -qq --no-install-recommends install \ - autoconf \ - automake \ - g++ \ - git \ - libboost-all-dev \ - libcap-dev \ - libcdb-dev \ - libedit-dev \ - libfstrm-dev \ - libgnutls28-dev \ - libh2o-evloop-dev \ - liblmdb-dev \ - libluajit-5.1-dev \ - libre2-dev \ - libsnmp-dev \ - libsodium-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - make \ - pkg-config \ - ragel \ - systemd \ - python3-venv - - install-clang-8 - - restore-cache-ccache: - product: dnsdist - - get-workspace - - run: - name: autoconf - command: BUILDER_VERSION=0.0.0-git1 autoreconf -vfi - working_directory: ~/project/pdns/dnsdistdist - - run: - name: configure - command: | - CC='clang-8' \ - CXX='clang++-8' \ - CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ - CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ - ./configure \ - --enable-option-checking=fatal \ - --enable-unit-tests \ - --enable-dnstap \ - --enable-dnscrypt \ - --enable-dns-over-tls \ - --enable-dns-over-https \ - --enable-systemd \ - --prefix=/opt/dnsdist \ - --with-gnutls \ - --with-libsodium \ - --with-lua=luajit \ - --with-libcap \ - --with-re2 \ - --enable-asan \ - --enable-ubsan - working_directory: ~/project/pdns/dnsdistdist - - run: - name: build - command: make -j1 -k - working_directory: ~/project/pdns/dnsdistdist - - save-ccache-cache: - product: dnsdist - - run: - name: Run unit tests - command: make check || (cat test-suite.log; false) - working_directory: ~/project/pdns/dnsdistdist - - run: - name: Install resulting binaries - command: make install - working_directory: ~/project/pdns/dnsdistdist - - persist_to_workspace: - root: /opt - paths: - - dnsdist - - build-dnsdist-with-tsan: - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - TSAN_OPTIONS: 'halt_on_error=1:suppressions=/opt/project/pdns/dnsdistdist/dnsdist-tsan.supp' - steps: - - run: - name: Install dependencies - command: | - apt-get update && apt-get -qq --no-install-recommends install \ - autoconf \ - automake \ - g++ \ - git \ - libboost-all-dev \ - libcap-dev \ - libcdb-dev \ - libedit-dev \ - libfstrm-dev \ - libgnutls28-dev \ - libh2o-evloop-dev \ - liblmdb-dev \ - libluajit-5.1-dev \ - libre2-dev \ - libsnmp-dev \ - libsodium-dev \ - libssl-dev \ - libsystemd-dev \ - libtool \ - make \ - pkg-config \ - ragel \ - systemd \ - python3-venv - - install-clang-8 - - restore-cache-ccache: - product: dnsdist-with-tsan - - get-workspace - - run: - name: autoconf - command: BUILDER_VERSION=0.0.0-git1 autoreconf -vfi - working_directory: ~/project/pdns/dnsdistdist - - run: - name: configure - command: | - CC='clang-8' \ - CXX='clang++-8' \ - CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ - CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ - ./configure \ - --enable-option-checking=fatal \ - --enable-unit-tests \ - --enable-dnstap \ - --enable-dnscrypt \ - --enable-dns-over-tls \ - --enable-dns-over-https \ - --enable-systemd \ - --prefix=/opt/dnsdist-with-tsan \ - --with-gnutls \ - --with-libsodium \ - --with-lua=luajit \ - --with-libcap \ - --with-re2 \ - --enable-tsan - working_directory: ~/project/pdns/dnsdistdist - - run: - name: build - command: make -j1 -k - working_directory: ~/project/pdns/dnsdistdist - - save-ccache-cache: - product: dnsdist-with-tsan - - run: - name: Run unit tests - command: make check || (cat test-suite.log; false) - working_directory: ~/project/pdns/dnsdistdist - - run: - name: Install resulting binaries - command: make install - working_directory: ~/project/pdns/dnsdistdist - - persist_to_workspace: - root: /opt - paths: - - dnsdist-with-tsan - - test-dnsdist-regression-with-tsan: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - TSAN_OPTIONS: 'halt_on_error=1:suppressions=/opt/project/pdns/dnsdistdist/dnsdist-tsan.supp' - steps: - - install-dnsdist-deps - - install-clang-8 # for the symbolizer - - get-workspace - - run: - name: setup snmp - command: | - apt-get -qq --no-install-recommends install snmpd - sed "s/agentxperms 0700 0755 dnsdist/agentxperms 0700 0755/g" regression-tests.dnsdist/snmpd.conf > /etc/snmp/snmpd.conf - /etc/init.d/snmpd start - - run: - name: install prometheus tools - command: | - apt-get -qq --no-install-recommends install prometheus - - run: - name: Run dnsdist tests - workdir: ~/project/regression-tests.dnsdist - command: | - DNSDISTBIN="/opt/dnsdist-with-tsan/bin/dnsdist" \ - ./runtests - - test-ixfrdist-regression: - resource_class: small - - docker: - - image: debian:buster - auth: - username: powerdnsreadonly - password: $DOCKERHUB_PASSWORD - environment: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - ASAN_OPTIONS: detect_leaks=0 - steps: - - install-ixfrdist-deps - - get-workspace - - run: - name: Run ixfrdist tests - workdir: ~/project/regression-tests.ixfrdist - command: | - IXFRDISTBIN="/opt/pdns-auth/bin/ixfrdist" \ - ./runtests - workflows: version: 2 coverity: @@ -1908,58 +1285,12 @@ workflows: branches: only: master jobs: - - coverity-auth: - context: auth-coverity - - coverity-dnsdist: - context: dnsdist-coverity - coverity-recursor: context: recursor-coverity build-and-test-all: jobs: - checkout - - build-auth: - requires: - - checkout - - test-auth-algorithms: - requires: - - build-auth - - test-auth-regress-odbc-sqlite3: - requires: - - build-auth - - test-auth-regress-odbc-mssql: - requires: - - build-auth - - test-auth-regress-gmysql: - requires: - - build-auth - - test-auth-regress-bind: - requires: - - build-auth - - test-auth-regress-geoip: - requires: - - build-auth - - test-auth-regress-lua2: - requires: - - build-auth - - test-auth-regress-gsqlite3: - requires: - - build-auth - - test-auth-regress-gpgsql: - requires: - - build-auth - - test-auth-regress-lmdb: - requires: - - build-auth - - test-auth-regress-ldap: - requires: - - build-auth - - test-auth-regress-tinydns: - requires: - - build-auth - - test-auth-py: - requires: - - build-auth - check-formatting: requires: - checkout @@ -1972,60 +1303,4 @@ workflows: - test-recursor-bulk: requires: - build-recursor - - build-dnsdist: - requires: - - checkout - - build-dnsdist-with-tsan: - requires: - - checkout - - test-dnsdist-regression-with-tsan: - requires: - - build-dnsdist-with-tsan - - test-ixfrdist-regression: - requires: - - build-auth - - build-docs: - jobs: - - checkout - - build-auth-docs: - filters: - branches: - ignore: master - requires: - - checkout - - build-recursor-docs: - filters: - branches: - ignore: master - requires: - - checkout - - build-dnsdist-docs: - filters: - branches: - ignore: master - requires: - - checkout - # These actually deploy - - deploy-auth-docs: - context: docs - filters: - branches: - only: master - requires: - - checkout - - deploy-recursor-docs: - context: docs - filters: - branches: - only: master - requires: - - checkout - - deploy-dnsdist-docs: - context: docs - filters: - branches: - only: master - requires: - - checkout diff --git a/.github/actions/spell-check/advice.md b/.github/actions/spell-check/advice.md index 2a32b6520781..c83423a8ef61 100644 --- a/.github/actions/spell-check/advice.md +++ b/.github/actions/spell-check/advice.md @@ -1,27 +1,25 @@ -
If you see a bunch of garbage +
If the flagged items do not appear to be text -If it relates to a ... -
well-formed pattern +If items relate to a ... +* well-formed pattern. -See if there's a [pattern](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples:-patterns) that would match it. + If you can write a [pattern](https://github.com/check-spelling/check-spelling/wiki/Configuration-Examples:-patterns) that would match it, + try adding it to the `patterns.txt` file. -If not, try writing one and adding it to the `patterns.txt` file. - -Patterns are Perl 5 Regular Expressions - you can [test]( + Patterns are Perl 5 Regular Expressions - you can [test]( https://www.regexplanet.com/advanced/perl/) yours before committing to verify it will match your lines. -Note that patterns can't match multiline strings. -
-
binary-ish string + Note that patterns can't match multiline strings. -Please add a file path to the `excludes.txt` file instead of just accepting the garbage. +* binary file. -File paths are Perl 5 Regular Expressions - you can [test]( + Please add a file path to the `excludes.txt` file matching the containing file. + + File paths are Perl 5 Regular Expressions - you can [test]( https://www.regexplanet.com/advanced/perl/) yours before committing to verify it will match your files. -`^` refers to the file's path from the root of the repository, so `^README\.md$` would exclude [README.md]( + `^` refers to the file's path from the root of the repository, so `^README\.md$` would exclude [README.md]( ../tree/HEAD/README.md) (on whichever branch you're using). -
diff --git a/.github/actions/spell-check/expect.txt b/.github/actions/spell-check/expect.txt index 78979a833607..3f057dae2894 100644 --- a/.github/actions/spell-check/expect.txt +++ b/.github/actions/spell-check/expect.txt @@ -77,6 +77,7 @@ Atomia aton attr atype +authanswers AUTHIP Authoritativedoc auths @@ -216,6 +217,7 @@ certusage cfea CFLAGS cgi +CGNAT changelog changeme changeset @@ -233,6 +235,7 @@ chroot chrooting CIDR classmethod +clientanswers Cloos closesocket clusions @@ -293,6 +296,7 @@ Cryptoki cryptopp cryptoshop css +csum csv csync ctime @@ -362,6 +366,7 @@ DIGESTALGOS Digitalus dijk dilinger +Dimitrios Directi Disqus distro @@ -462,6 +467,7 @@ ecs ECSDA ecswho edb +EDE editline edns ednsbufsiz @@ -515,6 +521,7 @@ Faerch faf failedservers failover +farsightsec favicon FBAE fbe @@ -560,6 +567,7 @@ frontend fstrm fullname fulltoc +fullycapable func Furnell Fusl @@ -682,6 +690,7 @@ hmac Hmi Hoentjen Hofstaedtler +Holger homepage Hooimeijer hostmaster @@ -732,6 +741,7 @@ Inno innodb inode installable +internic interop interoperability interoperation @@ -952,6 +962,7 @@ mariadb Markmann Massar matchtype +Mavrommatis maxdepth MAXINT maxlistdepth @@ -1067,6 +1078,7 @@ Neue Neuf newcontent nextval +nghttp nginx nic nimber @@ -1074,6 +1086,7 @@ Nixu nkey nmg Nncqx +NNNN noaction noad noall @@ -1155,6 +1168,7 @@ openssl opensuse openwall Opmeer +OPNUM optcode Opteron optmem @@ -1190,6 +1204,7 @@ pathconfig pathto pawal Pbackend +PCache pcap PCAPFILE pdf @@ -1448,6 +1463,7 @@ scopebits scopemask scriptable scriptlets +scrypt sdb sdfoijdfio sdig @@ -1508,6 +1524,7 @@ sigs SIGUSR singlethreaded Sipek +siphash sizeof Sjoerd slapd @@ -1569,6 +1586,7 @@ srv sshfp ssi ssl +SSLKEYLOGFILE sslmode sslrootcert SSQ @@ -1859,8 +1877,10 @@ Wzs Xander xchacha xdb +XDP Xek Xeon +XForwarded xfr xhtml xml @@ -1901,6 +1921,7 @@ zonetransfer Zonneveld zsk zskroll +ztc Zumstrull Zwane zzyzz diff --git a/.github/actions/spell-check/patterns.txt b/.github/actions/spell-check/patterns.txt index bb4f53b9045e..927645883c22 100644 --- a/.github/actions/spell-check/patterns.txt +++ b/.github/actions/spell-check/patterns.txt @@ -57,5 +57,10 @@ C0FFEE # tar arguments \b(?:)tar(?:\s+-[a-zA-Z]+|\s[a-z]+)+ -# DoT: DNS over TLS is often combined with other words (images, links, ...) +# DoT: DNS over TLS is often combined with other words (images, links, ...), same for DoH DoT +DoH + +# Contributors with non-ascii characters in their name +Hoffst[^[:ascii:]]+tte +Gri[^[:ascii:]] diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml index 59aafd3abe36..b656833b8818 100644 --- a/.github/workflows/build-and-test-all.yml +++ b/.github/workflows/build-and-test-all.yml @@ -8,39 +8,16 @@ on: - cron: '0 22 * * 3' jobs: - build-auth: - name: build auth - runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' - ASAN_OPTIONS: detect_leaks=0 - steps: - - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: recursive - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv install-clang - - run: inv install-auth-build-deps - - run: inv ci-autoconf - - run: inv ci-auth-configure - - run: inv ci-auth-make - # FIXME: save ccache here? - - run: inv ci-auth-install-remotebackend-ruby-deps - - run: inv ci-auth-run-unit-tests - - run: inv ci-make-install - - name: Store the binaries - uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar - with: - name: pdns-auth - path: /opt/pdns-auth - build-recursor: name: build recursor runs-on: ubuntu-20.04 + strategy: + matrix: + sanitizers: [ubsan+asan, tsan] env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" ASAN_OPTIONS: detect_leaks=0 + SANITIZERS: ${{ matrix.sanitizers }} defaults: run: working-directory: ./pdns/recursordist/ @@ -49,6 +26,17 @@ jobs: with: fetch-depth: 5 submodules: recursive + - name: get timestamp for cache + id: get-stamp + run: | + echo "::set-output name=stamp::$(/bin/date +%s)" + shell: bash + - name: let GitHub cache our ccache data + uses: actions/cache@v2 + with: + path: ~/.ccache + key: recursor-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }} + restore-keys: recursor-${{ matrix.sanitizers }}-ccache- - run: ../../build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - run: inv apt-fresh - run: inv install-clang @@ -56,76 +44,26 @@ jobs: - run: inv ci-autoconf - run: inv ci-rec-configure - run: inv ci-rec-make - # FIXME: save ccache here? - run: inv ci-rec-run-unit-tests - run: inv ci-make-install + - run: ccache -s - name: Store the binaries uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar with: - name: pdns-recursor + name: pdns-recursor-${{ matrix.sanitizers }} path: /opt/pdns-recursor + retention-days: 1 - build-dnsdist: - name: build dnsdist - runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' - ASAN_OPTIONS: detect_leaks=0 - defaults: - run: - working-directory: ./pdns/dnsdistdist/ - steps: - - uses: actions/checkout@v2.3.4 - with: - fetch-depth: 5 - submodules: recursive - - run: ../../build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv apt-fresh - - run: inv install-clang - - run: inv install-dnsdist-build-deps - - run: inv ci-autoconf - - run: inv ci-dnsdist-configure - - run: inv ci-dnsdist-make - # FIXME: save ccache here? - - run: inv ci-dnsdist-run-unit-tests - - run: inv ci-make-install - - name: Store the binaries - uses: actions/upload-artifact@v2 # this takes 30 seconds, maybe we want to tar - with: - name: dnsdist - path: /opt/dnsdist - - test-auth-api: - needs: build-auth + test-recursor-api: + needs: build-recursor runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' - ASAN_OPTIONS: detect_leaks=0 strategy: matrix: - include: - - backend: gsqlite3 - image: coscale/docker-sleep - - backend: gmysql - image: mysql:5 - - backend: gpgsql - image: postgres:9 - - backend: lmdb - image: coscale/docker-sleep - fail-fast: false - services: - database: - image: ${{ matrix.image }} - env: - POSTGRES_USER: runner - POSTGRES_HOST_AUTH_METHOD: trust - MYSQL_ALLOW_EMPTY_PASSWORD: 1 - ports: - - 3306:3306 - - 5432:5432 - # FIXME: this works around dist-upgrade stopping all docker containers. dist-upgrade is huge on these images anyway. Perhaps we do want to run our tasks in a Docker container too. - options: >- - --restart always + sanitizers: [ubsan+asan, tsan] + env: + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp" + ASAN_OPTIONS: detect_leaks=0 + TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp" steps: - uses: actions/checkout@v2.3.4 with: @@ -134,46 +72,24 @@ jobs: - name: Fetch the binaries uses: actions/download-artifact@v2 with: - name: pdns-auth - path: /opt/pdns-auth - # - name: Setup upterm session - # uses: lhotari/action-upterm@v1 + name: pdns-recursor-${{ matrix.sanitizers }} + path: /opt/pdns-recursor - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv add-auth-repo # FIXME: do we need this for rec API testing? - run: inv install-clang-runtime - - run: inv install-auth-test-deps -b ${{ matrix.backend }} - - run: inv test-api auth -b ${{ matrix.backend }} + - run: inv install-rec-test-deps + - run: inv test-api recursor - test-auth-backend: - needs: build-auth + test-recursor-regression: + needs: build-recursor runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' - ASAN_OPTIONS: detect_leaks=0 strategy: matrix: - include: - - backend: remote - image: coscale/docker-sleep - # - backend: gmysql - # image: mysql:5 - # - backend: gpgsql - # image: postgres:9 - # - backend: lmdb - # image: coscale/docker-sleep - fail-fast: false - services: - database: - image: ${{ matrix.image }} - env: - POSTGRES_USER: runner - POSTGRES_HOST_AUTH_METHOD: trust - MYSQL_ALLOW_EMPTY_PASSWORD: 1 - ports: - - 3306:3306 - - 5432:5432 - # FIXME: this works around dist-upgrade stopping all docker containers. dist-upgrade is huge on these images anyway. Perhaps we do want to run our tasks in a Docker container too. - options: >- - --restart always + sanitizers: [ubsan+asan] + env: + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp' + ASAN_OPTIONS: detect_leaks=0 + TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp" steps: - uses: actions/checkout@v2.3.4 with: @@ -182,22 +98,28 @@ jobs: - name: Fetch the binaries uses: actions/download-artifact@v2 with: - name: pdns-auth - path: /opt/pdns-auth - # - name: Setup upterm session - # uses: lhotari/action-upterm@v1 - # FIXME: install recursor for backends that have ALIAS + name: pdns-recursor-${{ matrix.sanitizers }} + path: /opt/pdns-recursor - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade + - run: inv add-auth-repo - run: inv install-clang-runtime - - run: inv install-auth-test-deps -b ${{ matrix.backend }} - - run: inv test-auth-backend -b ${{ matrix.backend }} + - run: inv install-rec-test-deps + - run: inv test-regression-recursor - test-recursor-api: + test-recursor-bulk: + name: 'test rec *mini* bulk' needs: build-recursor runs-on: ubuntu-20.04 + strategy: + matrix: + sanitizers: [ubsan+asan] + threads: [1, 2, 3, 4, 8, 16] + mthreads: [2048, 4096] + shards: [1, 2, 1024] env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' + UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp' ASAN_OPTIONS: detect_leaks=0 + TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp" steps: - uses: actions/checkout@v2.3.4 with: @@ -206,34 +128,32 @@ jobs: - name: Fetch the binaries uses: actions/download-artifact@v2 with: - name: pdns-recursor + name: pdns-recursor-${{ matrix.sanitizers }} path: /opt/pdns-recursor - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv add-auth-repo # FIXME: do we need this for rec API testing? - run: inv install-clang-runtime - - run: inv install-rec-test-deps - - run: inv test-api recursor + - run: inv install-rec-bulk-deps + - run: inv test-bulk-recursor ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }} - test-dnsdist-regression: - needs: build-dnsdist + collect: + needs: + - build-recursor + - test-recursor-api + - test-recursor-regression + - test-recursor-bulk runs-on: ubuntu-20.04 - env: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=/home/runner/work/pdns/pdns/build-scripts/UBSan.supp' - ASAN_OPTIONS: detect_leaks=0 steps: - uses: actions/checkout@v2.3.4 with: fetch-depth: 5 submodules: recursive - - name: Fetch the binaries - uses: actions/download-artifact@v2 - with: - name: dnsdist - path: /opt/dnsdist - - run: build-scripts/gh-actions-setup-inv # this runs apt update+upgrade - - run: inv install-clang-runtime - - run: inv install-dnsdist-test-deps - - run: inv test-dnsdist - + - name: Install yq + run: sudo wget https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq + - name: Get full list of jobs for this workflow + run: yq e '.jobs | keys' .github/workflows/build-and-test-all.yml | grep -v '^- collect' | sort | tee /tmp/workflow-jobs-list.yml + - name: Get list of jobs the collect job depends on + run: yq e '.jobs.collect.needs | ... comments=""' .github/workflows/build-and-test-all.yml | sort | tee /tmp/workflow-collect-dependencies.yml + - name: Diff them + run: diff -u /tmp/workflow-jobs-list.yml /tmp/workflow-collect-dependencies.yml # FIXME: if we can make upload/download-artifact fasts, running unit tests outside of build can let regression tests start earlier diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml index ce32ad0db014..09045b9ecd2e 100644 --- a/.github/workflows/builder.yml +++ b/.github/workflows/builder.yml @@ -2,10 +2,8 @@ name: 'Test package building for specific distributions' on: - push: - pull_request: schedule: - - cron: '0 22 * * 2' + - cron: '0 1 * * *' jobs: build: @@ -14,16 +12,28 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - product: ['authoritative', 'recursor', 'dnsdist'] + product: ['recursor'] os: - centos-7 - ubuntu-bionic - - oraclelinux-8 + - el-8 - centos-8-stream + - debian-bullseye + - ubuntu-jammy + fail-fast: false steps: - uses: actions/checkout@v2.3.4 with: - fetch-depth: 5 + fetch-depth: 0 # for correct version numbers submodules: recursive # this builds packages and runs our unit test (make check) - run: builder/build.sh -v -m ${{ matrix.product }} ${{ matrix.os }} + - name: Get version number + run: 'echo ::set-output name=version::$(readlink builder/tmp/latest)' + id: getversion + - name: Upload packages + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.product }}-${{ matrix.os }}-${{ steps.getversion.outputs.version }} + path: built_pkgs/ + retention-days: 7 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4223620ecdab..cdaf72292ae4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -82,6 +82,7 @@ jobs: liblmdb-dev \ liblua5.3-dev \ libmaxminddb-dev \ + libnghttp2-dev \ libp11-kit-dev \ libpq-dev \ libre2-dev \ @@ -99,7 +100,7 @@ jobs: if: matrix.product == 'auth' run: | autoreconf -vfi - ./configure --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe random remote tinydns' --enable-tools --enable-ixfrdist --enable-dns-over-tls --enable-experimental-pkcs11 --with-libsodium --enable-lua-records CFLAGS='-O0' CXXFLAGS='-O0' + ./configure --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' --enable-tools --enable-ixfrdist --enable-dns-over-tls --enable-experimental-pkcs11 --with-libsodium --enable-lua-records CFLAGS='-O0' CXXFLAGS='-O0' make -j8 -C ext make -j8 -C modules make -j8 -C pdns diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3cbc31d6c4fa..09c3dd9049bc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,10 +2,8 @@ name: 'Build Docker images' on: - push: - pull_request: schedule: - - cron: '0 22 * * 3' + - cron: '0 4 * * *' jobs: build: diff --git a/.github/workflows/secpoll.yml b/.github/workflows/secpoll.yml index 545a9ed48466..cb1151792dca 100644 --- a/.github/workflows/secpoll.yml +++ b/.github/workflows/secpoll.yml @@ -15,9 +15,10 @@ jobs: with: fetch-depth: 5 submodules: recursive + - run: sh docs/secpoll-check.sh docs/secpoll.zone - run: echo 'deb [arch=amd64] http://repo.powerdns.com/ubuntu focal-auth-master main' | sudo tee /etc/apt/sources.list.d/pdns.list - run: "echo -ne 'Package: pdns-*\nPin: origin repo.powerdns.com\nPin-Priority: 600\n' | sudo tee /etc/apt/preferences.d/pdns" - - run: curl https://repo.powerdns.com/CBC8B383-pub.asc | sudo apt-key add - + - run: sudo curl https://repo.powerdns.com/CBC8B383-pub.asc -o /etc/apt/trusted.gpg.d/CBC8B383-pub.asc - run: sudo apt-get update - run: sudo systemctl mask pdns - run: sudo apt-get install -y pdns-server pdns-backend-sqlite3 diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml new file mode 100644 index 000000000000..7cf39614917a --- /dev/null +++ b/.github/workflows/spelling.yml @@ -0,0 +1,16 @@ +# spelling.yml is disabled per https://github.com/check-spelling/check-spelling/security/advisories/GHSA-g86g-chm8-7r2p +name: Workflow should not run! +on: + push: + branches: '' + +jobs: + placeholder: + name: Should be disabled + runs-on: ubuntu-latest + if: false + steps: + - name: Task + run: | + echo 'Running this task would be bad' + exit 1 diff --git a/.not-formatted b/.not-formatted index 6a2ad40a943a..fb2b51ceabb0 100644 --- a/.not-formatted +++ b/.not-formatted @@ -68,7 +68,6 @@ ./pdns/dnsdist-lua-inspection.cc ./pdns/dnsdist-lua-rules.cc ./pdns/dnsdist-lua-vars.cc -./pdns/dnsdist-lua.cc ./pdns/dnsdist-lua.hh ./pdns/dnsdist-protobuf.cc ./pdns/dnsdist-protobuf.hh @@ -155,7 +154,6 @@ ./pdns/dynloader.cc ./pdns/dynmessenger.cc ./pdns/dynmessenger.hh -./pdns/ednscookies.cc ./pdns/ednsoptions.cc ./pdns/ednsoptions.hh ./pdns/ednspadding.cc @@ -243,20 +241,7 @@ ./pdns/query-local-address.hh ./pdns/rcpgenerator.cc ./pdns/rcpgenerator.hh -./pdns/rec-carbon.cc -./pdns/rec-lua-conf.cc -./pdns/rec-lua-conf.hh -./pdns/rec-snmp.cc -./pdns/rec-snmp.hh -./pdns/rec_channel.cc -./pdns/rec_channel.hh -./pdns/rec_channel_rec.cc -./pdns/rec_control.cc ./pdns/receiver.cc -./pdns/recpacketcache.cc -./pdns/recpacketcache.hh -./pdns/recursor_cache.cc -./pdns/recursor_cache.hh ./pdns/remote_logger.cc ./pdns/remote_logger.hh ./pdns/resolve-context.hh @@ -270,8 +255,6 @@ ./pdns/saxfr.cc ./pdns/sdig.cc ./pdns/secpoll-auth.cc -./pdns/secpoll-recursor.cc -./pdns/secpoll-recursor.hh ./pdns/secpoll.cc ./pdns/secpoll.hh ./pdns/serialtweaker.cc @@ -334,7 +317,6 @@ ./pdns/test-packetcache_hh.cc ./pdns/test-proxy_protocol_cc.cc ./pdns/test-rcpgenerator_cc.cc -./pdns/test-recpacketcache_cc.cc ./pdns/test-sha_hh.cc ./pdns/test-sholder_hh.cc ./pdns/test-signers.cc @@ -361,8 +343,6 @@ ./pdns/unix_utility.cc ./pdns/utility.hh ./pdns/uuid-utils.cc -./pdns/validate-recursor.cc -./pdns/validate-recursor.hh ./pdns/validate.cc ./pdns/validate.hh ./pdns/version.cc @@ -373,8 +353,6 @@ ./pdns/ws-api.hh ./pdns/ws-auth.cc ./pdns/ws-auth.hh -./pdns/ws-recursor.cc -./pdns/ws-recursor.hh ./pdns/xpf.cc ./pdns/xpf.hh ./pdns/zone2json.cc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28d3d7c3708b..4e0d4f77b159 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,8 +14,7 @@ If there is no similar issue, feature request or you're not sure, open a new issue. ## Filing a Feature Request -When filing a feature request, please start your issue title with "Feature request:", -this allows for quick distinguishing between issues and these requests. +When filing a feature request, please use the Feature request template provided. Please be as elaborate as possible when describing the feature you need. Provide at least the following information (if they are relevant): @@ -40,12 +39,14 @@ information: * Observed behavior: what actually happened when following the steps? * Relevant logs: Please use code blocks (\`\`\`) to format console output, logs, and code as it's very hard to read otherwise. +We provide convenient templates that make it easy to not forget any of these steps. + If you have already looked deeper into the problem, provide what you found as well. # Filing a Pull Request Code contributions are sent as a pull request on [GitHub](https://github.com/PowerDNS/pdns/pulls). -By submitting a Pull Request you agree to your code become GPLv2 licensed. +By submitting a Pull Request you agree to your code becoming GPLv2 licensed. ## Pull Request Guidelines A pull request, at the least, should have: @@ -57,12 +58,13 @@ A pull request, at the least, should have: And must: * Be filed against the master branch before any release branch -* Pass all tests in Travis +* Pass all tests in our CI (currently Github Actions and CircleCI) Information on the tests can be found in the repository at [/regression-tests/README.md](https://github.com/PowerDNS/pdns/blob/master/regression-tests/README.md) -and -[/regression-tests.recursor/README.md](https://github.com/PowerDNS/pdns/blob/master/regression-tests.recursor/README.md). +, +[/regression-tests.recursor/README.md](https://github.com/PowerDNS/pdns/blob/master/regression-tests.recursor/README.md), +plus various other directories with `regression-tests.*` names. ## Commit Guidelines * Tell why the change does what it does, not how it does it. @@ -72,10 +74,16 @@ and * Do not put whitespace fixes/cleanup and functionality changes in the same commit # Coding Guidelines -At the moment there is no established coding guideline, but here are some -general guidelines: +We have `clang-format` in place, but not for all files yet. +This is an incremental process. +If you're adding new code, adhering to the formatting config is appreciated. +Formatting breakage in already formatted files will be caught by the CI. +To format all files that are supposed to be formatted, run `make format-code` in the root of the tree. + +Additional guidelines: * Don't have end-of-line whitespace * Use spaces instead of tabs -* Stick to the style of the file you're editing -* Functions and classes must have a [docblock](http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html) +* Although the codebase does not consistently have them, [docblock](https://www.doxygen.nl/manual/docblocks.html)s on functions and classes are appreciated +* Never hesitate to write comments on anything that might not be immediately clear just from reading the code +* When adding whole new things, consider putting them in a `pdns::X` namespace. Look for `namespace pdns` in the codebase for examples. \ No newline at end of file diff --git a/Docker-README.md b/Docker-README.md index e2dc91a09d7e..3e7008cb3c15 100644 --- a/Docker-README.md +++ b/Docker-README.md @@ -42,3 +42,16 @@ In a plain Docker or Compose setup, this can be done by using the host PID names We have a Docker Compose example at https://github.com/PowerDNS/pdns/blob/master/docker-compose.yml . It brings up all three services, and exposes them to eachother by name (using Docker's internal DNS). In the dockerdata dir, you can find an example dnsdist Lua config (with Python helper to make DNS lookups non-blocking for dnsdist) for managing your auth/rec backends by name. + +# Privileged ports + +The default configurations included for dnsdist, Auth and Recursor attempt to bind to port 53, which may not be permitted by the platform on which you intend to use these images. Kubernetes clusters, for example, might have a restriction on binding to privileged ports unless the `NET_BIND_SERVICE` capability is explicitly added to the container's security context. + +There are multiple ways of dealing with these restrictions if you encounter them: + +* Grant the `NET_BIND_SERVICE` capability to the containers which utilize these images +* Use custom configuration files to bind to alternate ports outside of the privileged range. This can be done via the following configuration settings: + * dnsdist: `setLocal()` + * Auth & Recursor: `local-address` and/or `local-port` + +Note: Docker Engine 20.10.0 (released december 2020) removed the need to set the `NET_BIND_SERVICE` capability when attempting to bind to a privileged port. \ No newline at end of file diff --git a/Dockerfile-auth b/Dockerfile-auth index 2cb17cca3dc9..576cffd910c4 100644 --- a/Dockerfile-auth +++ b/Dockerfile-auth @@ -50,7 +50,7 @@ RUN mkdir /build && \ --with-lua=${LUAVER} \ --sysconfdir=/etc/powerdns \ --enable-option-checking=fatal \ - --with-dynmodules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe random remote tinydns' \ + --with-dynmodules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \ --enable-tools \ --enable-ixfrdist \ --with-unixodbc-lib=/usr/lib/$(dpkg-architecture -q DEB_BUILD_GNU_TYPE) && \ @@ -78,10 +78,9 @@ RUN apt-get install -y python3 python3-jinja2 sqlite3 tini libcap2-bin vim-tiny # Output from builder COPY --from=builder /build / RUN chmod 1777 /tmp # FIXME: better not use /build/tmp for equivs at all -RUN setcap 'cap_net_bind_service=+eip' /usr/local/sbin/pdns_server # Ensure dependencies are present -RUN apt install -y /tmp/equivs-dummy_1.0_all.deb && apt clean +RUN apt-get install -y /tmp/equivs-dummy_1.0_all.deb && apt-get clean # Start script COPY dockerdata/startup.py /usr/local/sbin/pdns_server-startup @@ -97,10 +96,10 @@ USER pdns # Set up database - this needs to be smarter RUN sqlite3 /var/lib/powerdns/pdns.sqlite3 < /usr/local/share/doc/pdns/schema.sqlite3.sql -# DNS ports +# Default DNS ports EXPOSE 53/udp EXPOSE 53/tcp -# webserver port +# Default webserver port EXPOSE 8081/tcp ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/sbin/pdns_server-startup"] diff --git a/Dockerfile-dnsdist b/Dockerfile-dnsdist index 521bc173dbd9..8406ada4384d 100644 --- a/Dockerfile-dnsdist +++ b/Dockerfile-dnsdist @@ -74,10 +74,9 @@ RUN apt-get install -y python3 python3-jinja2 python3-atomicwrites tini libcap2- # Output from builder COPY --from=builder /build / RUN chmod 1777 /tmp # FIXME: better not use /build/tmp for equivs at all -RUN setcap 'cap_net_bind_service=+eip' /usr/local/bin/dnsdist # Ensure dependencies are present -RUN apt install -y /tmp/equivs-dummy_1.0_all.deb && apt clean +RUN apt-get install -y /tmp/equivs-dummy_1.0_all.deb && apt-get clean # Config RUN mkdir -p /etc/dnsdist/conf.d /etc/dnsdist/templates.d @@ -91,12 +90,12 @@ RUN adduser --system --disabled-password --disabled-login --no-create-home --gro RUN chown pdns:pdns /etc/dnsdist/conf.d /etc/dnsdist/templates.d USER pdns -# DNS ports +# Default DNS ports EXPOSE 53/udp EXPOSE 53/tcp -# console port +# Default console port EXPOSE 5199/tcp -# webserver port +# Default webserver port EXPOSE 8083/tcp WORKDIR /etc/dnsdist diff --git a/Dockerfile-recursor b/Dockerfile-recursor index beee2fd9efe2..c6e17bef4004 100644 --- a/Dockerfile-recursor +++ b/Dockerfile-recursor @@ -77,10 +77,9 @@ RUN apt-get install -y python3 python3-jinja2 tini libcap2-bin && apt-get clean # Executables from builder COPY --from=builder /build / RUN chmod 1777 /tmp # FIXME: better not use /build/tmp for equivs at all -RUN setcap 'cap_net_bind_service=+eip' /usr/local/sbin/pdns_recursor # Ensure dependencies are present -RUN apt install -y /tmp/equivs-dummy_1.0_all.deb && apt clean +RUN apt-get install -y /tmp/equivs-dummy_1.0_all.deb && apt-get clean # Start script COPY dockerdata/startup.py /usr/local/sbin/pdns_recursor-startup @@ -96,11 +95,11 @@ RUN adduser --system --disabled-password --disabled-login --no-create-home --gro RUN chown pdns:pdns /var/run/pdns-recursor /etc/powerdns/recursor.d /etc/powerdns/templates.d USER pdns -# DNS ports +# Default DNS ports EXPOSE 53/udp EXPOSE 53/tcp -# webserver port +# Default webserver port EXPOSE 8082/tcp ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/sbin/pdns_recursor-startup"] diff --git a/build-scripts/docker/repo-test/generate-repo-files.py b/build-scripts/docker/repo-test/generate-repo-files.py index 1aba3b3cbb30..7ce2ce6483be 100755 --- a/build-scripts/docker/repo-test/generate-repo-files.py +++ b/build-scripts/docker/repo-test/generate-repo-files.py @@ -43,12 +43,14 @@ def init_argparser(): 'test PowerDNS repositories.') parser.add_argument('release', metavar='RELEASE', choices=[# Authoritative Server - 'auth-42', 'auth-43', 'auth-44', 'auth-master', + 'auth-43', 'auth-44', 'auth-45', 'auth-46', + 'auth-master', # Recursor - 'rec-42', 'rec-43', 'rec-44', 'rec-45', + 'rec-43', 'rec-44', 'rec-45', 'rec-46', 'rec-master', # DNSDist - 'dnsdist-15', 'dnsdist-16', 'dnsdist-master' + 'dnsdist-15', 'dnsdist-16', 'dnsdist-17', + 'dnsdist-master' ], help='the release to generate Docker files for: ' + '%(choices)s') @@ -139,29 +141,37 @@ def write_release_files (release): if release in ['auth-43', 'auth-master']: write_dockerfile('centos', '6', release) - if release in ['auth-41', 'auth-42', 'auth-43', 'auth-44', 'auth-master', - 'rec-42', 'rec-43', 'rec-44', 'rec-45', 'rec-master', - 'dnsdist-15', 'dnsdist-16', 'dnsdist-master']: + if release in ['auth-43', 'auth-44', 'auth-45', 'auth-46', 'auth-master', + 'rec-43', 'rec-44', 'rec-45', 'rec-46', 'rec-master', + 'dnsdist-15', 'dnsdist-16', 'dnsdist-17', 'dnsdist-master']: write_dockerfile('centos', '7', release) write_dockerfile('ubuntu', 'bionic', release) write_list_file('ubuntu', 'bionic', release) write_pkg_pin_file(release) - if release in ['auth-42', 'auth-43', 'auth-44', 'auth-master', - 'rec-42', 'rec-43', 'rec-44', 'rec-45', 'rec-master', - 'dnsdist-15', 'dnsdist-16', 'dnsdist-master']: + if release in ['auth-43', 'rec-43', 'dnsdist-15']: + write_dockerfile('raspbian', 'buster', release) + write_list_file('raspbian', 'buster', release) + + if release in ['auth-43', 'auth-44', 'auth-45', 'auth-46', 'auth-master', + 'rec-43', 'rec-44', 'rec-45', 'rec-46', 'rec-master', + 'dnsdist-15', 'dnsdist-16', 'dnsdist-17', 'dnsdist-master']: write_dockerfile('centos', '8', release) write_dockerfile('debian', 'buster', release) - write_dockerfile('raspbian', 'buster', release) write_list_file('debian', 'buster', release) - write_list_file('raspbian', 'buster', release) - if release in ['auth-43', 'auth-44', 'auth-master', - 'rec-43', 'rec-44', 'rec-45', 'rec-master', - 'dnsdist-15', 'dnsdist-16', 'dnsdist-master']: + if release in ['auth-43', 'auth-44', 'auth-45', 'auth-46', 'auth-master', + 'rec-43', 'rec-44', 'rec-45', 'rec-46', 'rec-master', + 'dnsdist-15', 'dnsdist-16', 'dnsdist-17', 'dnsdist-master']: write_dockerfile('ubuntu', 'focal', release) write_list_file('ubuntu', 'focal', release) + if release in ['auth-46', 'auth-master', + 'rec-46', 'rec-master', + 'dnsdist-16', 'dnsdist-17', 'dnsdist-master']: + write_dockerfile('debian', 'bullseye', release) + write_list_file('debian', 'bullseye', release) + # Test Release Functions diff --git a/build-scripts/docker/repo-test/templates/Dockerfile-debian.jinja2 b/build-scripts/docker/repo-test/templates/Dockerfile-debian.jinja2 index fea16594d52e..031db4e51b81 100644 --- a/build-scripts/docker/repo-test/templates/Dockerfile-debian.jinja2 +++ b/build-scripts/docker/repo-test/templates/Dockerfile-debian.jinja2 @@ -1,7 +1,6 @@ FROM {{ os_image }}:{{ os_version }} -RUN apt-get update -RUN apt-get install -y curl gnupg dnsutils apt-transport-https +RUN apt-get update && apt-get install -y curl gnupg dnsutils apt-transport-https {% if release.startswith('dnsdist-') %} COPY pkg-pin /etc/apt/preferences.d/dnsdist @@ -11,10 +10,9 @@ COPY pkg-pin /etc/apt/preferences.d/pdns COPY pdns.list.{{ release }}.{{ os }}-{{ os_version }} /etc/apt/sources.list.d/pdns.list -RUN curl https://repo.powerdns.com/FD380FBB-pub.asc | apt-key add - -RUN curl https://repo.powerdns.com/CBC8B383-pub.asc | apt-key add - -RUN apt-get update -RUN apt-get install -y {{ pkg }} +RUN curl https://repo.powerdns.com/CBC8B383-pub.asc -o /etc/apt/trusted.gpg.d/CBC8B383-pub.asc \ + https://repo.powerdns.com/FD380FBB-pub.asc -o /etc/apt/trusted.gpg.d/FD380FBB-pub.asc +RUN apt-get update && apt-get install -y {{ pkg }} {# in the old script this was just for rec-43, -44 and -45 #} {% if release.startswith('rec-') %} diff --git a/build-scripts/gh-actions-setup-inv b/build-scripts/gh-actions-setup-inv index de9bf0c4b349..02e5a231e237 100755 --- a/build-scripts/gh-actions-setup-inv +++ b/build-scripts/gh-actions-setup-inv @@ -2,4 +2,4 @@ sudo apt-get update sudo apt-get -qq -y dist-upgrade sudo apt-get -qq -y --no-install-recommends install python3-pip -sudo pip3 install git+https://github.com/pyinvoke/invoke@faa5728a6f76199a3da1750ed952e7efee17c1da \ No newline at end of file +sudo pip3 install git+https://github.com/pyinvoke/invoke@faa5728a6f76199a3da1750ed952e7efee17c1da diff --git a/build-scripts/test-recursor b/build-scripts/test-recursor index 1107e499d0ea..ac41da9da3f9 100755 --- a/build-scripts/test-recursor +++ b/build-scripts/test-recursor @@ -59,7 +59,7 @@ sleep 3 ./clean.sh cd ../regression-tests.recursor-dnssec -./runtests || EXIT=1 +./runtests $@ || EXIT=1 ./printlogs.py || true exit $EXIT diff --git a/builder-support/debian/authoritative/debian-buster/pdns-server.install b/builder-support/debian/authoritative/debian-buster/pdns-server.install index 74350bd6276b..480243ecc246 100644 --- a/builder-support/debian/authoritative/debian-buster/pdns-server.install +++ b/builder-support/debian/authoritative/debian-buster/pdns-server.install @@ -3,5 +3,4 @@ usr/bin/pdns_control usr/bin/pdnsutil usr/bin/zone2json usr/bin/zone2sql -usr/lib/*/pdns/librandombackend.so* usr/sbin/pdns_server diff --git a/builder-support/debian/authoritative/debian-buster/rules b/builder-support/debian/authoritative/debian-buster/rules index 50d6ffe03376..5dd51f643b52 100755 --- a/builder-support/debian/authoritative/debian-buster/rules +++ b/builder-support/debian/authoritative/debian-buster/rules @@ -10,7 +10,7 @@ include /usr/share/dpkg/default.mk export PDNS_TEST_NO_IPV6=1 # Backends -backends := bind ldap lmdb lua2 pipe gmysql godbc gpgsql gsqlite3 geoip remote random tinydns +backends := bind ldap lmdb lua2 pipe gmysql godbc gpgsql gsqlite3 geoip remote tinydns %: diff --git a/builder-support/debian/dnsdist/debian-buster/control b/builder-support/debian/dnsdist/debian-buster/control index 132d767e623d..624f518e52b0 100644 --- a/builder-support/debian/dnsdist/debian-buster/control +++ b/builder-support/debian/dnsdist/debian-buster/control @@ -14,6 +14,7 @@ Build-Depends: debhelper (>= 10), liblmdb-dev, libluajit-5.1-dev [!arm64 !s390x], liblua5.3-dev [arm64 s390x], + libnghttp2-dev, libre2-dev, libsnmp-dev, libsodium-dev, diff --git a/builder-support/debian/recursor/debian-buster/control b/builder-support/debian/recursor/debian-buster/control index f7dfb30e7ad9..dcc305e440b6 100644 --- a/builder-support/debian/recursor/debian-buster/control +++ b/builder-support/debian/recursor/debian-buster/control @@ -7,6 +7,7 @@ Build-Depends: debhelper (>= 10), dh-autoreconf, libboost-all-dev, libcap-dev, + libcurl4-openssl-dev, libluajit-5.1-dev [!arm64 !s390x], liblua5.3-dev [arm64 s390x], libfstrm-dev, diff --git a/builder-support/dockerfiles/Dockerfile.target.centos-7 b/builder-support/dockerfiles/Dockerfile.target.centos-7 index 7e5032cd4bb8..9deb58b99c6d 100644 --- a/builder-support/dockerfiles/Dockerfile.target.centos-7 +++ b/builder-support/dockerfiles/Dockerfile.target.centos-7 @@ -3,10 +3,10 @@ # This defines the distribution base layer # Put only the bare minimum of common commands here, without dev tools -@IF [ ${BUILDER_TARGET} = centos-7 ] +@IF [ ${BUILDER_TARGET} = centos-7 -o ${BUILDER_TARGET} = el-7 ] FROM centos:7 as dist-base @ENDIF -@IF [ ${BUILDER_TARGET} = centos-7-amd64 ] +@IF [ ${BUILDER_TARGET} = centos-7-amd64 -o ${BUILDER_TARGET} = el-7-amd64] FROM amd64/centos:7 as dist-base @ENDIF diff --git a/builder-support/dockerfiles/Dockerfile.target.centos-8 b/builder-support/dockerfiles/Dockerfile.target.centos-8 index ce5bd3bd90bf..ae0e4c9ab932 100644 --- a/builder-support/dockerfiles/Dockerfile.target.centos-8 +++ b/builder-support/dockerfiles/Dockerfile.target.centos-8 @@ -3,9 +3,12 @@ # This defines the distribution base layer # Put only the bare minimum of common commands here, without dev tools -@IF [ ${BUILDER_TARGET} = centos-8 -o ${BUILDER_TARGET} = centos-8-stream ] +@IF [ ${BUILDER_TARGET} = centos-8 ] FROM centos:8 as dist-base @ENDIF +@IF [ ${BUILDER_TARGET} = centos-8-stream ] +FROM quay.io/centos/centos:stream8 as dist-base +@ENDIF @IF [ ${BUILDER_TARGET} = centos-8-amd64 ] FROM amd64/centos:8 as dist-base @ENDIF diff --git a/builder-support/dockerfiles/Dockerfile.target.el-7 b/builder-support/dockerfiles/Dockerfile.target.el-7 new file mode 120000 index 000000000000..c6fc1ef94f7c --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.el-7 @@ -0,0 +1 @@ +Dockerfile.target.centos-7 \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.el-7-amd64 b/builder-support/dockerfiles/Dockerfile.target.el-7-amd64 new file mode 120000 index 000000000000..33697056909d --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.el-7-amd64 @@ -0,0 +1 @@ +Dockerfile.target.centos-7-amd64 \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.el-8 b/builder-support/dockerfiles/Dockerfile.target.el-8 new file mode 120000 index 000000000000..608942e5e532 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.el-8 @@ -0,0 +1 @@ +Dockerfile.target.oraclelinux-8 \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.el-8-amd64 b/builder-support/dockerfiles/Dockerfile.target.el-8-amd64 new file mode 120000 index 000000000000..f5f60eea19d2 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.el-8-amd64 @@ -0,0 +1 @@ +Dockerfile.target.oraclelinux-8-amd64 \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.el-8-arm64 b/builder-support/dockerfiles/Dockerfile.target.el-8-arm64 new file mode 120000 index 000000000000..55a25a393c26 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.el-8-arm64 @@ -0,0 +1 @@ +Dockerfile.target.oraclelinux-8-arm64 \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8 b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8 index d490101eb8c5..63dac11e52e8 100644 --- a/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8 +++ b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8 @@ -3,7 +3,16 @@ # This defines the distribution base layer # Put only the bare minimum of common commands here, without dev tools +@IF [ ${BUILDER_TARGET} = oraclelinux-8 -o ${BUILDER_TARGET} = el-8 ] FROM oraclelinux:8 as dist-base +@ENDIF +@IF [ ${BUILDER_TARGET} = oraclelinux-8-amd64 -o ${BUILDER_TARGET} = el-8-amd64 ] +FROM amd64/oraclelinux:8 as dist-base +@ENDIF +@IF [ ${BUILDER_TARGET} = oraclelinux-8-arm64 -o ${BUILDER_TARGET} = el-8-arm64 ] +FROM arm64v8/oraclelinux:8 as dist-base +@ENDIF + ARG BUILDER_CACHE_BUSTER= RUN touch /var/lib/rpm/* && dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ dnf install -y 'dnf-command(config-manager)' yum && \ diff --git a/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8-amd64 b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8-amd64 new file mode 120000 index 000000000000..608942e5e532 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8-amd64 @@ -0,0 +1 @@ +Dockerfile.target.oraclelinux-8 \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8-arm64 b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8-arm64 new file mode 120000 index 000000000000..608942e5e532 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.oraclelinux-8-arm64 @@ -0,0 +1 @@ +Dockerfile.target.oraclelinux-8 \ No newline at end of file diff --git a/builder-support/dockerfiles/Dockerfile.target.ubuntu-jammy b/builder-support/dockerfiles/Dockerfile.target.ubuntu-jammy new file mode 100644 index 000000000000..d623a956e2f6 --- /dev/null +++ b/builder-support/dockerfiles/Dockerfile.target.ubuntu-jammy @@ -0,0 +1,35 @@ +# First do the source builds +@INCLUDE Dockerfile.target.sdist + +@IF [ ${BUILDER_TARGET} = ubuntu-jammy ] +FROM ubuntu:jammy as dist-base +@ENDIF +@IF [ ${BUILDER_TARGET} = ubuntu-jammy-amd64 ] +FROM amd64/ubuntu:jammy as dist-base +@ENDIF +@IF [ ${BUILDER_TARGET} = ubuntu-jammy-arm64 ] +FROM arm64v8/ubuntu:jammy as dist-base +@ENDIF +ARG BUILDER_CACHE_BUSTER= +ARG APT_URL +RUN apt-get update && apt-get -y dist-upgrade + +@INCLUDE Dockerfile.debbuild-prepare + +@IF [ -n "$M_authoritative$M_all" ] +ADD builder-support/debian/authoritative/debian-buster/ pdns-${BUILDER_VERSION}/debian/ +@ENDIF + +@IF [ -n "$M_recursor$M_all" ] +ADD builder-support/debian/recursor/debian-buster/ pdns-recursor-${BUILDER_VERSION}/debian/ +@ENDIF + +@IF [ -n "$M_dnsdist$M_all" ] +ADD builder-support/debian/dnsdist/debian-buster/ dnsdist-${BUILDER_VERSION}/debian/ +@ENDIF + +@INCLUDE Dockerfile.debbuild + +# Do a test install and verify +# Can be skipped with skiptests=1 in the environment +# @EXEC [ "$skiptests" = "" ] && include Dockerfile.debtest diff --git a/builder-support/specs/dnsdist.spec b/builder-support/specs/dnsdist.spec index 24eaa3e5606e..97cdb7fdbf71 100644 --- a/builder-support/specs/dnsdist.spec +++ b/builder-support/specs/dnsdist.spec @@ -21,6 +21,7 @@ BuildRequires: systemd-devel BuildRequires: boost-devel BuildRequires: gnutls-devel BuildRequires: libcap-devel +BuildRequires: libnghttp2-devel BuildRequires: lmdb-devel BuildRequires: libsodium-devel %ifarch aarch64 diff --git a/builder-support/specs/pdns-recursor.spec b/builder-support/specs/pdns-recursor.spec index 812cb5329db5..b806482df399 100644 --- a/builder-support/specs/pdns-recursor.spec +++ b/builder-support/specs/pdns-recursor.spec @@ -17,6 +17,7 @@ BuildRequires: openssl-devel BuildRequires: net-snmp-devel BuildRequires: libsodium-devel BuildRequires: fstrm-devel +BuildRequires: libcurl-devel %ifarch aarch64 BuildRequires: lua-devel @@ -72,6 +73,14 @@ sed -i \ -e 's/# setgid=/setgid=pdns-recursor/' \ %{buildroot}%{_sysconfdir}/%{name}/recursor.conf +# The EL7 and 8 systemd actually supports %t, but its version number is older than that, so we do use seperate runtime dirs, but don't rely on RUNTIME_DIRECTORY +%if 0%{?rhel} < 9 +sed -e 's!/pdns_recursor!& --socket-dir=%t/pdns-recursor!' -i %{buildroot}/%{_unitdir}/pdns-recursor.service +%if 0%{?rhel} < 8 +sed -e 's!/pdns_recursor!& --socket-dir=%t/pdns-recursor-%i!' -e 's!RuntimeDirectory=pdns-recursor!&-%i!' -i %{buildroot}/%{_unitdir}/pdns-recursor@.service +%endif +%endif + %pre getent group pdns-recursor > /dev/null || groupadd -r pdns-recursor getent passwd pdns-recursor > /dev/null || \ diff --git a/builder-support/specs/pdns.spec b/builder-support/specs/pdns.spec index 3dc25bcfec29..86bdc752d8e6 100644 --- a/builder-support/specs/pdns.spec +++ b/builder-support/specs/pdns.spec @@ -190,7 +190,7 @@ export CPPFLAGS="-DLDAP_DEPRECATED" --disable-silent-rules \ --with-modules='' \ --with-lua=%{lua_implementation} \ - --with-dynmodules='%{backends} random' \ + --with-dynmodules='%{backends}' \ --enable-tools \ --with-libsodium \ %if 0%{?amzn} != 2 @@ -225,6 +225,14 @@ chmod 600 %{buildroot}%{_sysconfdir}/%{name}/pdns.conf %{__mv} %{buildroot}/%{_bindir}/zone2ldap %{buildroot}/%{_bindir}/pdns-zone2ldap %{__mv} %{buildroot}/%{_mandir}/man1/zone2ldap.1 %{buildroot}/%{_mandir}/man1/pdns-zone2ldap.1 +# The EL7 and 8 systemd actually supports %t, but its version number is older than that, so we do use seperate runtime dirs, but don't rely on RUNTIME_DIRECTORY +%if 0%{?rhel} < 9 +sed -e 's!/pdns_server!& --socket-dir=%t/pdns!' -i %{buildroot}/%{_unitdir}/pdns.service +%if 0%{?rhel} < 8 +sed -e 's!/pdns_server!& --socket-dir=%t/pdns-%i!' -e 's!RuntimeDirectory=pdns!&-%i!' -i %{buildroot}/%{_unitdir}/pdns@.service +%endif +%endif + %check PDNS_TEST_NO_IPV6=1 make %{?_smp_mflags} -C pdns check || (cat pdns/test-suite.log && false) @@ -265,7 +273,6 @@ systemctl daemon-reload ||: %{_bindir}/zone2json %{_bindir}/zone2sql %{_libdir}/%{name}/libbindbackend.so -%{_libdir}/%{name}/librandombackend.so %{_mandir}/man1/pdns-zone2ldap.1.gz %{_mandir}/man1/pdns_control.1.gz %{_mandir}/man1/pdns_server.1.gz @@ -379,4 +386,4 @@ systemctl daemon-reload ||: %{_sysconfdir}/%{name}/ixfrdist.example.yml %{_unitdir}/ixfrdist.service %{_unitdir}/ixfrdist@.service -%endif \ No newline at end of file +%endif diff --git a/configure.ac b/configure.ac index af50e9927f29..249f2cdb3988 100644 --- a/configure.ac +++ b/configure.ac @@ -169,9 +169,9 @@ AC_ARG_WITH([socketdir], [socketdir="$withval"] ) -modules="bind gmysql random" +modules="bind gmysql" AC_ARG_WITH([modules], - [AS_HELP_STRING([--with-modules], [which backends to compile with @<:@default=bind gmysql random@:>@])], + [AS_HELP_STRING([--with-modules], [which backends to compile with @<:@default=bind gmysql@:>@])], [modules="$withval"] ) @@ -350,7 +350,6 @@ AC_CONFIG_FILES([ modules/lmdbbackend/Makefile modules/lua2backend/Makefile modules/pipebackend/Makefile - modules/randombackend/Makefile modules/remotebackend/Makefile modules/tinydnsbackend/Makefile ]) diff --git a/contrib/ProtobufLogger.py b/contrib/ProtobufLogger.py index 812863376e6f..27884d07ae8b 100644 --- a/contrib/ProtobufLogger.py +++ b/contrib/ProtobufLogger.py @@ -98,7 +98,41 @@ def getAppliedPolicyTypeAsString(polType): elif polType == dnsmessage_pb2.PBDNSMessage.NSIP: return 'NS IP' + @staticmethod + def getEventAsString(event): + descr = dnsmessage_pb2.PBDNSMessage.DESCRIPTOR + return descr.EnumValueName('EventType', event); + + @staticmethod + def getTransportAsString(transport): + descr = dnsmessage_pb2.PBDNSMessage.DESCRIPTOR + return descr.EnumValueName('SocketProtocol', transport); + def printResponse(self, message): + if message.trace: + print("- Event Trace:") + for event in message.trace: + ev = self.getEventAsString(event.event) + if event.event == dnsmessage_pb2.PBDNSMessage.CustomEvent and event.HasField('custom'): + ev += ":" + event.custom + ev += '(' + str(event.ts) + valstr = '' + if event.HasField('boolVal'): + valstr = str(event.boolVal) + elif event.HasField('intVal'): + valstr = str(event.intVal) + elif event.HasField('stringVal'): + valstr = event.stringVal + elif event.HasField('bytesVal'): + valstr = binascii.hexlify(event.bytesVal) + if len(valstr) > 0: + valstr = ',' + valstr + if not event.start: + startstr = ',done' + else: + startstr = '' + print("\t- %s%s%s)" % (ev, valstr, startstr)) + if message.HasField('response'): response = message.response @@ -173,10 +207,7 @@ def printSummary(self, msg, typestr): if msg.HasField('to'): iptostr = '[' + socket.inet_ntop(socket.AF_INET6, msg.to) + ']' - if msg.socketProtocol == dnsmessage_pb2.PBDNSMessage.UDP: - protostr = 'UDP' - else: - protostr = 'TCP' + protostr = self.getTransportAsString(msg.socketProtocol) if msg.HasField('fromPort'): fromportstr = ':' + str(msg.fromPort) + ' ' diff --git a/contrib/xdp-filter.ebpf.src b/contrib/xdp-filter.ebpf.src new file mode 100644 index 000000000000..5ae8a84020d4 --- /dev/null +++ b/contrib/xdp-filter.ebpf.src @@ -0,0 +1,430 @@ +#include +#include +#include +#include + +#define DNS_PORT 53 + +// do not use libc includes because this causes clang +// to include 32bit headers on 64bit ( only ) systems. +typedef __u8 uint8_t; +typedef __u16 uint16_t; +typedef __u32 uint32_t; +typedef __u64 uint64_t; +#define memcpy __builtin_memcpy + +/* + * Helper pointer to parse the incoming packets + * Copyright 2020, NLnet Labs, All rights reserved. + */ +struct cursor { + void *pos; + void *end; +}; + +/* + * Store the VLAN header + * Copyright 2020, NLnet Labs, All rights reserved. + */ +struct vlanhdr { + uint16_t tci; + uint16_t encap_proto; +}; + +/* + * Store the DNS header + * Copyright 2020, NLnet Labs, All rights reserved. + */ +struct dnshdr { + uint16_t id; + union { + struct { +#if BYTE_ORDER == LITTLE_ENDIAN + uint8_t rd : 1; + uint8_t tc : 1; + uint8_t aa : 1; + uint8_t opcode : 4; + uint8_t qr : 1; + + uint8_t rcode : 4; + uint8_t cd : 1; + uint8_t ad : 1; + uint8_t z : 1; + uint8_t ra : 1; +#elif BYTE_ORDER == BIG_ENDIAN || BYTE_ORDER == PDP_ENDIAN + uint8_t qr : 1; + uint8_t opcode : 4; + uint8_t aa : 1; + uint8_t tc : 1; + uint8_t rd : 1; + + uint8_t ra : 1; + uint8_t z : 1; + uint8_t ad : 1; + uint8_t cd : 1; + uint8_t rcode : 4; +#endif + } as_bits_and_pieces; + uint16_t as_value; + } flags; + uint16_t qdcount; + uint16_t ancount; + uint16_t nscount; + uint16_t arcount; +}; + +/* + * Store the qname and qtype + */ +struct dns_qname +{ + uint8_t qname[255]; + uint16_t qtype; +}; + +/* + * The possible actions to perform on the packet + * PASS: XDP_PASS + * DROP: XDP_DROP + * TC: set TC bit and XDP_TX + */ +enum dns_action : uint8_t { + PASS = 0, + DROP = 1, + TC = 2 +}; + +/* + * Store the matching counter and the associated action for a blocked element + */ +struct map_value +{ + uint64_t counter; + enum dns_action action; +}; + +BPF_TABLE_PINNED("hash", uint32_t, struct map_value, v4filter, 1024, "/sys/fs/bpf/dnsdist/addr-v4"); +BPF_TABLE_PINNED("hash", struct in6_addr, struct map_value, v6filter, 1024, "/sys/fs/bpf/dnsdist/addr-v6"); +BPF_TABLE_PINNED("hash", struct dns_qname, struct map_value, qnamefilter, 1024, "/sys/fs/bpf/dnsdist/qnames"); + +/* + * Initializer of a cursor pointer + * Copyright 2020, NLnet Labs, All rights reserved. + */ +static inline void cursor_init(struct cursor *c, struct xdp_md *ctx) +{ + c->end = (void *)(long)ctx->data_end; + c->pos = (void *)(long)ctx->data; +} + +/* + * Header parser functions + * Copyright 2020, NLnet Labs, All rights reserved. + */ +#define PARSE_FUNC_DECLARATION(STRUCT) \ +static inline struct STRUCT *parse_ ## STRUCT (struct cursor *c) \ +{ \ + struct STRUCT *ret = c->pos; \ + if (c->pos + sizeof(struct STRUCT) > c->end) \ + return 0; \ + c->pos += sizeof(struct STRUCT); \ + return ret; \ +} + +PARSE_FUNC_DECLARATION(ethhdr) +PARSE_FUNC_DECLARATION(vlanhdr) +PARSE_FUNC_DECLARATION(iphdr) +PARSE_FUNC_DECLARATION(ipv6hdr) +PARSE_FUNC_DECLARATION(udphdr) +PARSE_FUNC_DECLARATION(dnshdr) + +/* + * Parse ethernet frame and fill the struct + * Copyright 2020, NLnet Labs, All rights reserved. + */ +static inline struct ethhdr *parse_eth(struct cursor *c, uint16_t *eth_proto) +{ + struct ethhdr *eth; + + if (!(eth = parse_ethhdr(c))) + return 0; + + *eth_proto = eth->h_proto; + if (*eth_proto == bpf_htons(ETH_P_8021Q) + || *eth_proto == bpf_htons(ETH_P_8021AD)) { + struct vlanhdr *vlan; + + if (!(vlan = parse_vlanhdr(c))) + return 0; + + *eth_proto = vlan->encap_proto; + if (*eth_proto == bpf_htons(ETH_P_8021Q) + || *eth_proto == bpf_htons(ETH_P_8021AD)) { + if (!(vlan = parse_vlanhdr(c))) + return 0; + + *eth_proto = vlan->encap_proto; + } + } + return eth; +} + +/* + * Recalculate the checksum + * Copyright 2020, NLnet Labs, All rights reserved. + */ +static inline void update_checksum(uint16_t *csum, uint16_t old_val, uint16_t new_val) +{ + uint32_t new_csum_value; + uint32_t new_csum_comp; + uint32_t undo; + + undo = ~((uint32_t)*csum) + ~((uint32_t)old_val); + new_csum_value = undo + (undo < ~((uint32_t)old_val)) + (uint32_t)new_val; + new_csum_comp = new_csum_value + (new_csum_value < ((uint32_t)new_val)); + new_csum_comp = (new_csum_comp & 0xFFFF) + (new_csum_comp >> 16); + new_csum_comp = (new_csum_comp & 0xFFFF) + (new_csum_comp >> 16); + *csum = (uint16_t)~new_csum_comp; +} + +/* + * Set the TC bit and swap UDP ports + * Copyright 2020, NLnet Labs, All rights reserved. + */ +static inline enum dns_action set_tc_bit(struct udphdr *udp, struct dnshdr *dns) +{ + uint16_t old_val = dns->flags.as_value; + + // change the DNS flags + dns->flags.as_bits_and_pieces.ad = 0; + dns->flags.as_bits_and_pieces.qr = 1; + dns->flags.as_bits_and_pieces.tc = 1; + + // change the UDP destination to the source + udp->dest = udp->source; + udp->source = bpf_htons(DNS_PORT); + + // calculate and write the new checksum + update_checksum(&udp->check, old_val, dns->flags.as_value); + + // bounce + return TC; +} + +/* + * Check DNS QName + * Returns PASS if message needs to go through (i.e. pass) + * TC if (modified) message needs to be replied + * DROP if message needs to be blocke + */ +static inline enum dns_action check_qname(struct cursor *c) +{ + struct dns_qname qkey = {0}; + uint8_t qname_byte; + uint16_t qtype; + int length = 0; + + for(int i = 0; i<255; i++) { + if (bpf_probe_read_kernel(&qname_byte, sizeof(qname_byte), c->pos)) { + return PASS; + } + c->pos += 1; + if (length == 0) { + if (qname_byte == 0 || qname_byte > 63 ) { + break; + } + length += qname_byte; + } else { + length--; + } + if (qname_byte >= 'A' && qname_byte <= 'Z') { + qkey.qname[i] = qname_byte + ('a' - 'A'); + } else { + qkey.qname[i] = qname_byte; + } + } + + // if the last read qbyte is not 0 incorrect QName format), return PASS + if (qname_byte != 0) { + return PASS; + } + + // get QType + if(bpf_probe_read_kernel(&qtype, sizeof(qtype), c->pos)) { + return PASS; + } + + struct map_value* value; + + // check if Qname/Qtype is blocked + qkey.qtype = bpf_htons(qtype); + value = qnamefilter.lookup(&qkey); + if (value) { + __sync_fetch_and_add(&value->counter, 1); + return value->action; + } + + // check with Qtype 255 (*) + qkey.qtype = 255; + + value = qnamefilter.lookup(&qkey); + if (value) { + __sync_fetch_and_add(&value->counter, 1); + return value->action; + } + + return PASS; +} + +/* + * Parse IPv4 DNS mesage. + * Returns PASS if message needs to go through (i.e. pass) + * TC if (modified) message needs to be replied + * DROP if message needs to be blocked + */ +static inline enum dns_action udp_dns_reply_v4(struct cursor *c, uint32_t key) +{ + struct udphdr *udp; + struct dnshdr *dns; + + if (!(udp = parse_udphdr(c)) || udp->dest != bpf_htons(DNS_PORT)) { + return PASS; + } + + // check that we have a DNS packet + if (!(dns = parse_dnshdr(c))) { + return PASS; + } + + // if the address is blocked, perform the corresponding action + struct map_value* value = v4filter.lookup(&key); + + if (value) { + __sync_fetch_and_add(&value->counter, 1); + if (value->action == TC) { + return set_tc_bit(udp, dns); + } else { + return value->action; + } + } else { + enum dns_action action = check_qname(c); + if (action == TC) { + return set_tc_bit(udp, dns); + } else { + return action; + } + } + + return PASS; +} + +/* + * Parse IPv6 DNS mesage. + * Returns PASS if message needs to go through (i.e. pass) + * TC if (modified) message needs to be replied + * DROP if message needs to be blocked + */ +static inline enum dns_action udp_dns_reply_v6(struct cursor *c, struct in6_addr key) +{ + struct udphdr *udp; + struct dnshdr *dns; + + + if (!(udp = parse_udphdr(c)) || udp->dest != bpf_htons(DNS_PORT)) { + return PASS; + } + + // check that we have a DNS packet + ; + if (!(dns = parse_dnshdr(c))) { + return PASS; + } + + // if the address is blocked, perform the corresponding action + struct map_value* value = v6filter.lookup(&key); + + if (value) { + __sync_fetch_and_add(&value->counter, 1); + if (value->action == TC) { + return set_tc_bit(udp, dns); + } else { + return value->action; + } + } else { + enum dns_action action = check_qname(c); + if (action == TC) { + return set_tc_bit(udp, dns); + } else { + return action; + } + } + + return PASS; +} + +int xdp_dns_filter(struct xdp_md *ctx) +{ + // store variables + struct cursor c; + struct ethhdr *eth; + uint16_t eth_proto; + struct iphdr *ipv4; + struct ipv6hdr *ipv6; + int r = 0; + + // initialise the cursor + cursor_init(&c, ctx); + + // pass the packet if it is not an ethernet one + if ((eth = parse_eth(&c, ð_proto))) { + // IPv4 packets + if (eth_proto == bpf_htons(ETH_P_IP)) + { + if (!(ipv4 = parse_iphdr(&c)) || bpf_htons(ipv4->protocol != IPPROTO_UDP)) { + return XDP_PASS; + } + // if TC bit must not be set, apply the action + if ((r = udp_dns_reply_v4(&c, bpf_htonl(ipv4->saddr))) != TC) { + return r == DROP ? XDP_DROP : XDP_PASS; + } + + // swap src/dest IP addresses + uint32_t swap_ipv4 = ipv4->daddr; + ipv4->daddr = ipv4->saddr; + ipv4->saddr = swap_ipv4; + } + // IPv6 packets + else if (eth_proto == bpf_htons(ETH_P_IPV6)) + { + ; + if (!(ipv6 = parse_ipv6hdr(&c)) || bpf_htons(ipv6->nexthdr != IPPROTO_UDP)) { + return XDP_PASS; + } + // if TC bit must not be set, apply the action + if ((r = udp_dns_reply_v6(&c, ipv6->saddr)) != TC) { + return r == DROP ? XDP_DROP : XDP_PASS; + } + + // swap src/dest IP addresses + struct in6_addr swap_ipv6 = ipv6->daddr; + ipv6->daddr = ipv6->saddr; + ipv6->saddr = swap_ipv6; + + } + // pass all non-IP packets + else { + return XDP_PASS; + } + } else { + return XDP_PASS; + } + + // swap MAC addresses + uint8_t swap_eth[ETH_ALEN]; + memcpy(swap_eth, eth->h_dest, ETH_ALEN); + memcpy(eth->h_dest, eth->h_source, ETH_ALEN); + memcpy(eth->h_source, swap_eth, ETH_ALEN); + + // bounce the request + return XDP_TX; +} diff --git a/contrib/xdp.py b/contrib/xdp.py new file mode 100644 index 000000000000..c84893ae8e11 --- /dev/null +++ b/contrib/xdp.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +from bcc import BPF +import ctypes as ct +import netaddr +import socket + +# Constants +QTYPES = {'LOC': 29, '*': 255, 'IXFR': 251, 'UINFO': 100, 'NSEC3': 50, 'AAAA': 28, 'CNAME': 5, 'MINFO': 14, 'EID': 31, 'GPOS': 27, 'X25': 19, 'HINFO': 13, 'CAA': 257, 'NULL': 10, 'DNSKEY': 48, 'DS': 43, 'ISDN': 20, 'SOA': 6, 'RP': 17, 'UID': 101, 'TALINK': 58, 'TKEY': 249, 'PX': 26, 'NSAP-PTR': 23, 'TXT': 16, 'IPSECKEY': 45, 'DNAME': 39, 'MAILA': 254, 'AFSDB': 18, 'SSHFP': 44, 'NS': 2, 'PTR': 12, 'SPF': 99, 'TA': 32768, 'A': 1, 'NXT': 30, 'AXFR': 252, 'RKEY': 57, 'KEY': 25, 'NIMLOC': 32, 'A6': 38, 'TLSA': 52, 'MG': 8, 'HIP': 55, 'NSEC': 47, 'GID': 102, 'SRV': 33, 'DLV': 32769, 'NSEC3PARAM': 51, 'UNSPEC': 103, 'TSIG': 250, 'ATMA': 34, 'RRSIG': 46, 'OPT': 41, 'MD': 3, 'NAPTR': 35, 'MF': 4, 'MB': 7, 'DHCID': 49, 'MX': 15, 'MAILB': 253, 'CERT': 37, 'NINFO': 56, 'APL': 42, 'MR': 9, 'SIG': 24, 'WKS': 11, 'KX': 36, 'NSAP': 22, 'RT': 21, 'SINK': 40} +INV_QTYPES = {v: k for k, v in QTYPES.items()} +ACTIONS = {1 : 'DROP', 2 : 'TC'} + +DROP_ACTION = 1 +TC_ACTION = 2 + +# The interface on wich the filter will be attached +DEV = "eth0" + +# The list of blocked IPv4, IPv6 and QNames +# IP format : (IPAddress, Action) +# QName format : (QName, QType, Action) +blocked_ipv4 = [("192.0.2.1", TC_ACTION)] +blocked_ipv6 = [("2001:db8::1", TC_ACTION)] +blocked_qnames = [("localhost", "A", DROP_ACTION), ("test.com", "*", TC_ACTION)] + +# Main +xdp = BPF(src_file="xdp-filter.ebpf.src") + +fn = xdp.load_func("xdp_dns_filter", BPF.XDP) +xdp.attach_xdp(DEV, fn, 0) + +v4filter = xdp.get_table("v4filter") +v6filter = xdp.get_table("v6filter") +qnamefilter = xdp.get_table("qnamefilter") + +for ip in blocked_ipv4: + print(f"Blocking {ip}") + key = v4filter.Key(int(netaddr.IPAddress(ip[0]).value)) + leaf = v4filter.Leaf() + leaf.counter = 0 + leaf.action = ip[1] + v4filter[key] = leaf + +for ip in blocked_ipv6: + print(f"Blocking {ip}") + ipv6_int = int(netaddr.IPAddress(ip[0]).value) + ipv6_bytes = bytearray([(ipv6_int & (255 << 8*(15-i))) >> (8*(15-i)) for i in range(16)]) + key = (ct.c_uint8 * 16).from_buffer(ipv6_bytes) + leaf = v6filter.Leaf() + leaf.counter = 0 + leaf.action = ip[1] + v6filter[key] = leaf + +for qname in blocked_qnames: + print(f"Blocking {qname}") + key = qnamefilter.Key() + qn = bytearray() + for sub in qname[0].split('.'): + qn.append(len(sub)) + for ch in sub: + qn.append(ord(ch)) + qn.extend((0,) * (255 - len(qn))) + key.qname = (ct.c_ubyte * 255).from_buffer(qn) + key.qtype = ct.c_uint16(QTYPES[qname[1]]) + leaf = qnamefilter.Leaf() + leaf.counter = 0 + leaf.action = qname[2] + qnamefilter[key] = leaf + +print("Filter is ready") +try: + xdp.trace_print() +except KeyboardInterrupt: + pass + +for item in v4filter.items(): + print(f"{str(netaddr.IPAddress(item[0].value))} ({ACTIONS[item[1].action]}): {item[1].counter}") +for item in v6filter.items(): + print(f"{str(socket.inet_ntop(socket.AF_INET6, item[0]))} ({ACTIONS[item[1].action]}): {item[1].counter}") +for item in qnamefilter.items(): + print(f"{''.join(map(chr, item[0].qname)).strip()}/{INV_QTYPES[item[0].qtype]} ({ACTIONS[item[1].action]}): {item[1].counter}") + +xdp.remove_xdp(DEV, 0) diff --git a/dockerdata/dnsdist-resolver.py b/dockerdata/dnsdist-resolver.py index 7344f5019b09..152298d61635 100755 --- a/dockerdata/dnsdist-resolver.py +++ b/dockerdata/dnsdist-resolver.py @@ -40,5 +40,4 @@ def run(self): lt.fname = '/tmp/dnsdist-resolver.out' lt.start() for line in sys.stdin: - print(line.split()) lt.targets=line.split() diff --git a/docs/appendices/backend-writers-guide.rst b/docs/appendices/backend-writers-guide.rst index 79bd11fba793..6659a3124e6a 100644 --- a/docs/appendices/backend-writers-guide.rst +++ b/docs/appendices/backend-writers-guide.rst @@ -263,10 +263,6 @@ The third part registers the RandomFactory with PowerDNS. This is a simple C++ trick which makes sure that this function is called on execution of the binary or when loading the dynamic module. -Please note that a RandomBackend is actually in most PowerDNS releases. -By default it lives on random.example.com, but you can change that by -setting :ref:`setting-random-hostname`. - .. note:: This simple backend neglects to handle case properly! diff --git a/docs/appendices/types.rst b/docs/appendices/types.rst index 1721b635461c..24220dfc93b7 100644 --- a/docs/appendices/types.rst +++ b/docs/appendices/types.rst @@ -141,6 +141,13 @@ HINFO Hardware Info record, used to specify CPU and operating system. Stored with a single space separating these two, example: 'i386 Linux'. +.. _types-https: + +HTTPS +----- + +See :ref:`SVCB ` for more information. + .. _types-key: KEY @@ -249,27 +256,10 @@ The stored format is: :: - primary hostmaster serial refresh retry expire default_ttl + primary hostmaster serial refresh retry expire minimum Besides the primary and the hostmaster, all fields are numerical. -PowerDNS has a set of default values: - -- primary: :ref:`setting-default-soa-name` - configuration option -- hostmaster: ``hostmaster@domain-name`` -- serial: 0 -- refresh: 10800 (3 hours) -- retry: 3600 (1 hour) -- expire: 604800 (1 week) -- default_ttl: 3600 (1 hour) - -The fields have complicated and sometimes controversial meanings. The -'serial' field is special. If left at 0, the default, PowerDNS will -perform an internal list of the domain to determine highest change_date -field of all records within the zone, and use that as the zone serial -number. This means that the serial number is always raised when changes -are made to the zone, as long as the change_date field is being set. -Make sure to check whether your backend of choice supports Autoserial. +The fields have complicated and sometimes controversial meanings. .. _types-spf: @@ -300,6 +290,26 @@ priority. For example, be encoded with ``0`` in the priority field and ``100 389 mars.conaxis.ch`` in the content field. +.. _types-svcb: + +SVCB, HTTPS +----------- +.. versionadded:: 4.4.0 + +SVCB records, defined in +(`draft-ietf-dnsop-svcb-https-07 +`__) +are used to facilitate the lookup of information needed to make +connections to network services. SVCB records allow a service to be +provided from multiple alternative endpoints, each with associated +parameters (such as transport protocol configuration and keys for +encrypting the TLS ClientHello). They also enable aliasing of apex +domains, which is not possible with CNAME. The HTTPS RR is a variation +of SVCB for HTTPS and HTTP origins. + +Additional processing is supported for these types. +Some :doc:`PowerDNS extensions <../guides/svcb>` for automatic IP address hints exist as well. + TKEY, TSIG ---------- diff --git a/docs/backends/bind.rst b/docs/backends/bind.rst index 1232c5df0279..04a110ebbb47 100644 --- a/docs/backends/bind.rst +++ b/docs/backends/bind.rst @@ -5,7 +5,6 @@ BIND zone file backend * Master: Yes * Slave: Yes * Superslave: Experimental -* Autoserial: No * DNSSEC: Yes * Disabled data: No * Comments: No diff --git a/docs/backends/generic-mysql.rst b/docs/backends/generic-mysql.rst index 9c19182e5ded..53d5fb974e80 100644 --- a/docs/backends/generic-mysql.rst +++ b/docs/backends/generic-mysql.rst @@ -5,7 +5,6 @@ Generic MySQL backend * Master: Yes * Slave: Yes * Superslave: Yes -* Autoserial: No * Case: All lower * DNSSEC: Yes (set ``gmysql-dnssec``) * Disabled data: Yes diff --git a/docs/backends/generic-odbc.rst b/docs/backends/generic-odbc.rst index 99fdef58a1ad..1f47d3e23b51 100644 --- a/docs/backends/generic-odbc.rst +++ b/docs/backends/generic-odbc.rst @@ -5,7 +5,6 @@ Generic ODBC Backend * Master: Yes * Slave: Yes * Superslave: Yes -* Autoserial: No * Case: All lower * DNSSEC: Yes * Disabled data: Yes diff --git a/docs/backends/generic-postgresql.rst b/docs/backends/generic-postgresql.rst index b726cbe466a9..92d75bd4e708 100644 --- a/docs/backends/generic-postgresql.rst +++ b/docs/backends/generic-postgresql.rst @@ -5,7 +5,6 @@ Generic PostgreSQL backend * Master: Yes * Slave: Yes * Superslave: Yes -* Autoserial: No * Case: All lower * DNSSEC: Yes (set ``gpgsql-dnssec``) * Disabled data: Yes diff --git a/docs/backends/index.rst b/docs/backends/index.rst index c068d202f51b..d72f52efbd85 100644 --- a/docs/backends/index.rst +++ b/docs/backends/index.rst @@ -3,35 +3,35 @@ Backends The following table describes the supported backends and some of their capabilities. -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| Name | Native | Master | Slave | Super slave | :doc:`DNSSEC <../dnssec/index>` | Launch | -+================================================+========+========+=======+==============+=================================+==============+ -| :doc:`BIND ` | Yes | Yes | Yes | Experimental | Yes | ``bind`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Generic Mysql ` | Yes | Yes | Yes | Yes | Yes | ``gmysql`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Generic ODBC ` | Yes | Yes | Yes | Yes | Yes | ``godbc`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Generic Postgresql ` | Yes | Yes | Yes | Yes | Yes | ``gpgsql`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Generic SQLite3 ` | Yes | Yes | Yes | Yes | Yes | ``gsqlite3`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`GeoIP ` | Yes | No | No | No | Yes | ``geoip`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`LDAP ` | Yes | Yes | No | No | No | ``ldap`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`LMDB ` | Yes | Yes | Yes | No | Yes | ``lmdb`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Lua2 ` | Yes | Yes | No | No | Yes | ``lua2`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Pipe ` | Yes | No | No | No | Partial | ``pipe`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Random ` | Yes | No | No | No | Partial | ``random`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`Remote ` | Yes | Yes\* | Yes\* | Yes\* | Yes\* | ``remote`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ -| :doc:`TinyDNS ` | Yes | Yes | No | No | Partial | ``tinydns`` | -+------------------------------------------------+--------+--------+-------+--------------+---------------------------------+--------------+ ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| Name | Native | Master | Slave | Super slave | :doc:`Dynamic DNS Update <../dnsupdate>` | :doc:`DNSSEC <../dnssec/index>` | Launch | ++================================================+========+========+=======+==============+==========================================+=================================+==============+ +| :doc:`BIND ` | Yes | Yes | Yes | Experimental | No | Yes | ``bind`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Generic Mysql ` | Yes | Yes | Yes | Yes | Yes | Yes | ``gmysql`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Generic ODBC ` | Yes | Yes | Yes | Yes | Yes | Yes | ``godbc`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Generic Postgresql ` | Yes | Yes | Yes | Yes | Yes | Yes | ``gpgsql`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Generic SQLite3 ` | Yes | Yes | Yes | Yes | Yes | Yes | ``gsqlite3`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`GeoIP ` | Yes | No | No | No | No | Yes | ``geoip`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`LDAP ` | Yes | Yes | No | No | No | No | ``ldap`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`LMDB ` | Yes | Yes | Yes | No | No | Yes | ``lmdb`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Lua2 ` | Yes | Yes | No | No | No | Yes | ``lua2`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Pipe ` | Yes | No | No | No | No | Partial | ``pipe`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Random ` | Yes | No | No | No | No | Partial | ``random`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`Remote ` | Yes | Yes\* | Yes\* | Yes\* | No | Yes\* | ``remote`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ +| :doc:`TinyDNS ` | Yes | Yes | No | No | No | Partial | ``tinydns`` | ++------------------------------------------------+--------+--------+-------+--------------+------------------------------------------+---------------------------------+--------------+ All the generic SQL backends have similar functionality, apart from the database they communicate with. These backends have :doc:`features unique ` to the generic SQL backends. diff --git a/docs/backends/ldap.rst b/docs/backends/ldap.rst index 14765f9dbf6d..4a5b29735c4a 100644 --- a/docs/backends/ldap.rst +++ b/docs/backends/ldap.rst @@ -5,7 +5,6 @@ LDAP backend * Master: Yes * Slave: No * Superslave: No -* Autoserial: No * DNSSEC: No * Disabled data: No * Comments: No diff --git a/docs/backends/lua2.rst b/docs/backends/lua2.rst index e41fa996b6cf..f0793318e6b1 100644 --- a/docs/backends/lua2.rst +++ b/docs/backends/lua2.rst @@ -5,7 +5,6 @@ Lua2 Backend * Master: Yes * Slave: No * Superslave: No -* Autoserial: No * DNSSEC: Yes * Disabled data: No * Comments: No diff --git a/docs/backends/pipe.rst b/docs/backends/pipe.rst index ddd01094a3c1..d857624bb464 100644 --- a/docs/backends/pipe.rst +++ b/docs/backends/pipe.rst @@ -5,7 +5,6 @@ Pipe Backend * Master: No * Slave: No * Superslave: No -* Autoserial: No * Case: Depends * DNSSEC: Partial, no delegation, no key storage * Disabled data: No diff --git a/docs/backends/random.rst b/docs/backends/random.rst index 6f7af49907a6..61c36011aa85 100644 --- a/docs/backends/random.rst +++ b/docs/backends/random.rst @@ -1,11 +1,13 @@ Random Backend ============== +.. deprecated:: 4.6.0 + This backend has been removed in 4.6.0 + - Native: Yes - Master: No - Slave: No - Superslave: No -- Autoserial: No - Case: Depends - DNSSEC: Yes, no key storage - Disabled data: No diff --git a/docs/backends/remote.rst b/docs/backends/remote.rst index e26fc16e777d..63ef3dad5cc4 100644 --- a/docs/backends/remote.rst +++ b/docs/backends/remote.rst @@ -5,7 +5,6 @@ Remote Backend * Master: Yes\* * Slave: Yes\* * Superslave: Yes\* -* Autoserial: Yes\* * DNSSEC: Yes\* * Zone caching: Yes\* * Multiple instances: Yes diff --git a/docs/backends/tinydns.rst b/docs/backends/tinydns.rst index ad788fc32da0..8e65e836e628 100644 --- a/docs/backends/tinydns.rst +++ b/docs/backends/tinydns.rst @@ -5,7 +5,6 @@ TinyDNS Backend - Master: Yes - Slave: No - Superslave: No -- Autoserial: No - DNSSEC: No - Zone caching: Yes - Multiple Instances: Yes diff --git a/docs/changelog/4.5.rst b/docs/changelog/4.5.rst index d79696330719..01eb72a9fc78 100644 --- a/docs/changelog/4.5.rst +++ b/docs/changelog/4.5.rst @@ -1,6 +1,50 @@ Changelogs for 4.5.x ==================== +.. changelog:: + :version: 4.5.2 + :released: 10th of November 2021 + + This is release 4.5.2 of the Authoritative Server. + It contains several robustness fixes for the bindbackend, and for SOA handling. + These fixes are especially important for zone cache users. + + .. change:: + :tags: Bug Fixes + :pullreq: 10968 + + bindbackend: skip rejected zones during list and search + + .. change:: + :tags: Bug Fixes + :pullreq: 10964 + + make the zone cache more robust for bad data and save some SOA queries for DNSSEC zones (Kees Monshouwer) + + .. change:: + :tags: Bug Fixes + :pullreq: 10962 + + api, check SOA location (Kees Monshouwer) + + .. change:: + :tags: Bug Fixes + :pullreq: 10952 + + improve dnsname exception handling for SOA records (Kees Monshouwer) + + .. change:: + :tags: Bug Fixes + :pullreq: 10792 + + improve SOA parse exception handling (Kees Monshouwer) + + .. change:: + :tags: Bug Fixes + :pullreq: 10778 + + try to reload rejected zones in bind-backend once every bind-check-interval (Kees Monshouwer) + .. changelog:: :version: 4.5.1 :released: 26th of July 2021 diff --git a/docs/changelog/4.6.rst b/docs/changelog/4.6.rst new file mode 100644 index 000000000000..1e00dec2ecdb --- /dev/null +++ b/docs/changelog/4.6.rst @@ -0,0 +1,112 @@ +Changelogs for 4.6.x +==================== + +.. changelog:: + :version: 4.6.0-alpha1 + :released: 7th of October 2021 + + This is version 4.6.0-alpha1 of the Authoritative Server. + This release contains a ton of improvements and bug fixes compared to 4.5, and a couple of user visible changes. + + New features: + + * support for incoming PROXY headers + * support for EDNS Cookies + + Please make sure to read the :doc:`upgrade notes <../upgrading>` before upgrading. + + .. change:: + :tags: Improvements + :pullreq: 10727 + + SVCB: Accept known SVCParams in generic format + + .. change:: + :tags: New Features + :pullreq: 10289 + + Implement EDNS cookies + + .. change:: + :tags: Improvements + :pullreq: 10653 + + Convert locks to LockGuarded + + .. change:: + :tags: Improvements + :pullreq: 10157 + + Move to hashed passwords for the web interface + + .. change:: + :tags: New Features + :pullreq: 10660 + + incoming PROXY support + + .. change:: + :tags: Improvements + :pullreq: 10484 + + remove randombackend (Kees Monshouwer) + + .. change:: + :tags: Bug Fixes + :pullreq: 10672 + + lmdb-safe: resizing while there might be open transactions is unsafe + + .. change:: + :tags: Bug Fixes + :pullreq: 10642 + + checkKey: handle NULL error string from OpenSSL more gracefully + + .. change:: + :tags: Improvements + :pullreq: 10780 + + remove attodot feature; fixes #10254 + + .. change:: + :tags: Bug Fixes + :pullreq: 10788 + + improve SOA parse exception handling (Kees Monshouwer) + + .. change:: + :tags: Improvements + :pullreq: 10562 + + auth NOTIFY: log names of NSes we could not resolve + + .. change:: + :tags: Improvements + :pullreq: 10779 + + pdns_control man page: fix copy+paste for decrypt (Josh Soref) + + .. change:: + :tags: Bug Fixes + :pullreq: 10748 + + try to reload rejected zones in bind-backend (Kees Monshouwer) + + .. change:: + :tags: Bug Fixes + :pullreq: 10231 + + send YXDOMAIN for too long DNAME synth + + .. change:: + :tags: Improvements + :pullreq: 10770 + + Gracefully handle uncaught exceptions in the UDP path + + .. change:: + :tags: Improvements + :pullreq: 10749 + + add cookie counters (Kees Monshouwer) diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst index 73a6df2b8078..88d003b43600 100644 --- a/docs/changelog/index.rst +++ b/docs/changelog/index.rst @@ -6,6 +6,7 @@ The changelogs for the PowerDNS Authoritative Server are split between release t .. toctree:: :maxdepth: 2 + 4.6 4.5 4.4 4.3 diff --git a/docs/common/api/configsetting.rst b/docs/common/api/configsetting.rst index 68aaad057de8..a72e736a805c 100644 --- a/docs/common/api/configsetting.rst +++ b/docs/common/api/configsetting.rst @@ -3,7 +3,7 @@ ConfigSetting .. json:object:: ConfigSetting - Represents a configuration item (as found in :doc:'../settings') + Represents a configuration item (as found in :doc:`../../settings`) :property string type: set to "ConfigSetting" :property string name: The name of this setting (e.g. 'webserver-port') diff --git a/docs/http-api/index.rst b/docs/http-api/index.rst index 34c01e4c3936..60b449fff924 100644 --- a/docs/http-api/index.rst +++ b/docs/http-api/index.rst @@ -16,7 +16,7 @@ The following webserver related configuration items are available: * :ref:`setting-webserver`: If set to anything but 'no', a webserver is launched. * :ref:`setting-webserver-address`: Address to bind the webserver to. Defaults to 127.0.0.1, which implies that only the local computer is able to connect to the nameserver! To allow remote hosts to connect, change to 0.0.0.0 or the physical IP address of your nameserver. -* :ref:`setting-webserver-password`: If set, viewers will have to enter this plaintext password in order to gain access to the statistics, in addition to entering the configured API key on the index page. +* :ref:`setting-webserver-password`: If set, viewers will have to enter this password in order to gain access to the statistics, in addition to entering the configured API key on the index page. * :ref:`setting-webserver-port`: Port to bind the webserver to. * :ref:`setting-webserver-allow-from`: Netmasks that are allowed to connect to the webserver * :ref:`setting-webserver-max-bodysize`: Maximum request/response body size in megabytes diff --git a/docs/lua-records/index.rst b/docs/lua-records/index.rst index dbde1185c2f3..dad201f082c6 100644 --- a/docs/lua-records/index.rst +++ b/docs/lua-records/index.rst @@ -177,6 +177,8 @@ separate records. The full example from above can also be written as:: "return ifurlup('https://www.lua.org/', " "{EUips, USAips}, settings) " ) +.. _lua-details-security: + Details & Security ------------------ LUA records are synthesized on query. They can also be transferred via AXFR diff --git a/docs/manpages/pdnsutil.1.rst b/docs/manpages/pdnsutil.1.rst index 9649979042fe..6c59f1651a8d 100644 --- a/docs/manpages/pdnsutil.1.rst +++ b/docs/manpages/pdnsutil.1.rst @@ -189,6 +189,11 @@ edit-zone *ZONE* **EDITOR** is empty, *pdnsutil* falls back to using *editor*. get-meta *ZONE* [*ATTRIBUTE*]... Get zone metadata. If no *ATTRIBUTE* given, lists all known. +hash-password [*WORK-FACTOR*] + This convenience command asks for a password and returns a hashed + and salted version, for use as a webserver password or api key. + An optional scrypt work factor can be specified, in power of two, + otherwise it defaults to 1024. hash-zone-record *ZONE* *RNAME* This convenience command hashes the name *RNAME* according to the NSEC3 settings of *ZONE*. Refuses to hash for zones with no NSEC3 @@ -265,7 +270,7 @@ ipencrypt *IP-ADDRESS* password Encrypt an IP address according to the 'ipcipher' standard ipdecrypt *IP-ADDRESS* password - Encrypt an IP address according to the 'ipcipher' standard + Decrypt an IP address according to the 'ipcipher' standard See also -------- diff --git a/docs/manpages/sdig.1.rst b/docs/manpages/sdig.1.rst index 9c6fad35f976..ee916ed50a39 100644 --- a/docs/manpages/sdig.1.rst +++ b/docs/manpages/sdig.1.rst @@ -57,6 +57,8 @@ tlsProvider *name* when using DoT, use TLS provider *name*. Currently supported (if compiled in): `openssl` and `gnutls`. Default is `openssl` if available. xpf *XPFCODE* *XPFVERSION* *XPFPROTO* *XPFSRC* *XPFDST* Send an *XPF* additional with these parameters. +opcode *OPNUM* + Use opcode *OPNUM* instead of 0 (Query). For example, ``sdig 192.0.2.1 53 example.com SOA opcode 4`` sends a ``NOTIFY``. Examples -------- diff --git a/docs/modes-of-operation.rst b/docs/modes-of-operation.rst index f15af66ffacc..810fe793c8da 100644 --- a/docs/modes-of-operation.rst +++ b/docs/modes-of-operation.rst @@ -12,9 +12,9 @@ Native replication Native replication is the default, unless other operation is specifically configured. Native replication basically means that -PowerDNS will not send out DNS update notifications, nor will react to -them. PowerDNS assumes that the backend is taking care of replication -unaided. +PowerDNS will not send out DNS update notifications, nor will it react +to them. PowerDNS assumes that the backend is taking care of +replication unaided. MySQL replication has proven to be very robust and well suited, even over transatlantic connections between badly peering ISPs. diff --git a/docs/performance.rst b/docs/performance.rst index d4e889bbe9c6..610b8965f678 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -1,9 +1,10 @@ Performance and Tuning ====================== -In general, best performance is achieved on recent Linux 4.x kernels and -using MySQL, although many of the largest PowerDNS installations are -based on PostgreSQL. FreeBSD also performs very well. +In general, best performance is achieved on recent Linux kernels with +the bindbackend, or if something more database-like is preferred, +the LMDB backend. Meanwhile many of the largest PowerDNS installations are +based on PostgreSQL or MySQL. Database servers can require configuration to achieve decent performance. It is especially worth noting that several vendors ship @@ -407,11 +408,17 @@ udp-do-queries ^^^^^^^^^^^^^^ Number of queries received with the DO (DNSSEC OK) bit set +.. _stat-udp-in-csum-errors: + +udp-in-csum-errors +^^^^^^^^^^^^^^^^^^ +Number of UDP packets received with an invalid checksum + .. _stat-udp-in-errors: udp-in-errors ^^^^^^^^^^^^^ -Number of packets, received faster than the OS could process them +Number of packets received faster than the OS could process them .. _stat-udp-noport-errors: @@ -467,12 +474,42 @@ udp6-answers ^^^^^^^^^^^^ Number of answers sent out over UDPv6 +.. _stat-udp6-in-csum-errors: + +udp6-in-csum-errors +^^^^^^^^^^^^^^^^^^^ +Number of IPv6 UDP packets received with an invalid checksum + +.. _stat-udp6-in-errors: + +udp6-in-errors +^^^^^^^^^^^^^^ +Number of IPv6 UDP packets received faster than the OS could process them + +.. _stat-udp6-noport-errors: + +udp6-noport-errors +^^^^^^^^^^^^^^^^^^ +Number of IPv6 UDP packets where an ICMP response was received that the remote port was not listening + .. _stat-udp6-queries: udp6-queries ^^^^^^^^^^^^ Number of questions received over UDPv6 +.. _stat-udp6-recvbuf-errors: + +udp6-recvbuf-errors +^^^^^^^^^^^^^^^^^^^ +Number of errors caused in the IPv6 UDP receive buffer + +.. _stat-udp6-sndbuf-errors: + +udp6-sndbuf-errors +^^^^^^^^^^^^^^^^^^ +Number of errors caused in the IPv6 UDP send buffer + .. _stat-uptime: uptime diff --git a/docs/requirements.txt b/docs/requirements.txt index 9cf30e4daacf..622263e53d84 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,4 +4,5 @@ git+https://github.com/pieterlexis/sphinx-jsondomain@no-type-links git+https://github.com/pieterlexis/sphinx-changelog@render-tags sphinxcontrib-fulltoc guzzle_sphinx_theme -docutils!=0.15 +docutils!=0.15,<0.18 +jinja2<3.1.0 diff --git a/docs/secpoll-check.sh b/docs/secpoll-check.sh new file mode 100644 index 000000000000..e95ce9cbbcf4 --- /dev/null +++ b/docs/secpoll-check.sh @@ -0,0 +1,12 @@ +#!/bin/sh +if [ $# -ne 1 ]; then + echo usage: $0 file + exit 1 +fi +if egrep -v '^(@|;|$)' "$1" | egrep -v 'ubuntu|debian|raspbian|fedora' | egrep -v '(auth|recursor|dnsdist)-[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)[0-9]+)?\.security-status +60 IN TXT "[1-3].*"' +then + echo Not OK + exit 1 +fi +echo OK +exit 0 diff --git a/docs/secpoll.zone b/docs/secpoll.zone index acae0cd5a9d1..b3df19187d77 100644 --- a/docs/secpoll.zone +++ b/docs/secpoll.zone @@ -1,4 +1,4 @@ -@ 86400 IN SOA pdns-public-ns1.powerdns.com. pieter\.lexis.powerdns.com. 2021073001 10800 3600 604800 10800 +@ 86400 IN SOA pdns-public-ns1.powerdns.com. pieter\.lexis.powerdns.com. 2021111601 10800 3600 604800 10800 @ 3600 IN NS pdns-public-ns1.powerdns.com. @ 3600 IN NS pdns-public-ns2.powerdns.com. @@ -84,6 +84,8 @@ auth-4.5.0-rc1.security-status 60 IN TXT "3 Unsupported auth-4.5.0-rc2.security-status 60 IN TXT "3 Unsupported pre-release (known vulnerabilities)" auth-4.5.0.security-status 60 IN TXT "3 Upgrade now, see https://doc.powerdns.com/authoritative/security-advisories/powerdns-advisory-2021-01.html" auth-4.5.1.security-status 60 IN TXT "1 OK" +auth-4.5.2.security-status 60 IN TXT "1 OK" +auth-4.6.0-alpha1.security-status 60 IN TXT "1 Unsupported pre-release" ; Auth Debian auth-3.4.1-2.debian.security-status 60 IN TXT "3 Upgrade now, see https://doc.powerdns.com/3/security/powerdns-advisory-2015-01/ and https://doc.powerdns.com/3/security/powerdns-advisory-2015-02/ and https://doc.powerdns.com/3/security/powerdns-advisory-2016-02/ and https://doc.powerdns.com/3/security/powerdns-advisory-2016-03/ and https://doc.powerdns.com/3/security/powerdns-advisory-2016-04/ and https://doc.powerdns.com/3/security/powerdns-advisory-2016-05/" @@ -260,6 +262,8 @@ recursor-4.4.2.security-status 60 IN TXT "1 OK" recursor-4.4.3.security-status 60 IN TXT "1 OK" recursor-4.4.4.security-status 60 IN TXT "1 OK" recursor-4.4.5.security-status 60 IN TXT "1 OK" +recursor-4.4.6.security-status 60 IN TXT "1 OK" +recursor-4.4.7.security-status 60 IN TXT "1 OK" recursor-4.5.0-alpha1.security-status 60 IN TXT "3 Unsupported pre-release" recursor-4.5.0-alpha2.security-status 60 IN TXT "3 Unsupported pre-release" recursor-4.5.0-alpha3.security-status 60 IN TXT "3 Unsupported pre-release" @@ -272,6 +276,11 @@ recursor-4.5.2.security-status 60 IN TXT "1 OK" recursor-4.5.3.security-status 60 IN TXT "2 Unsupported pre-release" recursor-4.5.4.security-status 60 IN TXT "1 OK" recursor-4.5.5.security-status 60 IN TXT "1 OK" +recursor-4.5.6.security-status 60 IN TXT "1 OK" +recursor-4.5.7.security-status 60 IN TXT "1 OK" +recursor-4.6.0-alpha1.security-status 60 IN TXT "1 Unsupported pre-release" +recursor-4.6.0-alpha2.security-status 60 IN TXT "1 Unsupported pre-release" +recursor-4.6.0-beta1.security-status 60 IN TXT "1 Unsupported pre-release" ; Recursor Debian recursor-3.6.2-2.debian.security-status 60 IN TXT "3 Upgrade now, see https://doc.powerdns.com/3/security/powerdns-advisory-2015-01/ and https://doc.powerdns.com/3/security/powerdns-advisory-2016-02/" @@ -413,3 +422,7 @@ dnsdist-1.6.0-alpha3.security-status 60 IN TXT "3 Unsuppor dnsdist-1.6.0-rc1.security-status 60 IN TXT "3 Unsupported pre-release" dnsdist-1.6.0-rc2.security-status 60 IN TXT "3 Unsupported pre-release" dnsdist-1.6.0.security-status 60 IN TXT "1 OK" +dnsdist-1.6.1.security-status 60 IN TXT "1 OK" +dnsdist-1.7.0-alpha1.security-status 60 IN TXT "1 Unsupported pre-release" +dnsdist-1.7.0-alpha2.security-status 60 IN TXT "1 Unsupported pre-release" +dnsdist-1.7.0-beta1.security-status 60 IN TXT "1 Unsupported pre-release" diff --git a/docs/security.rst b/docs/security.rst index d6428d8d7018..ac3b7cddac51 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -25,6 +25,11 @@ Set these parameters immediately if they are not set! Jailing the process in a chroot ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Modern Linux distributions, with systemd for process management, do a better job of constraining PowerDNS than chroot can. +We strongly suggest using distribution/OS features for process containment instead of the :ref:`setting-chroot` option. +The text below is kept for those users that have specific reasons to prefer chroot. +chroot functionality is not actively tested during development and might break during upgrades. + The :ref:`setting-chroot` option secures PowerDNS to its own directory so that even if it should become compromised and under control of external influences, it will have a hard time affecting the rest of the system. Even though this will hamper hackers a lot, chroot jails have been known to be broken. @@ -34,7 +39,7 @@ Even though this will hamper hackers a lot, chroot jails have been known to be b socket which should live within the chroot. It is often possible to hardlink such a socket into the chroot dir. -When running with master or slave support, be aware that many operating +When running with primary or secondary support, be aware that many operating systems need access to specific libraries (often ``/lib/libnss*``) in order to support resolution of domain names! You can also hardlink these. @@ -60,3 +65,20 @@ Separation will enhance your database security highly. Recommended. .. _securitypolling: .. include:: common/secpoll.rst + +Trusting zone files +------------------- +In some scenarios the PowerDNS server must handle zone files coming from an untrusted third party. +For these cases, it is recommended to take extra protective measures in addition to the measures above: + +- Set :ref:`setting-max-generate-steps` to a low number, this will limit the amount of resources used by rogue ``$GENERATE`` templates. +- Set :ref:`setting-max-include-depth` to ``0``, this will disallow the ``$INCLUDE`` directive, avoiding problems with include loops and related issues. +- Set :ref:`setting-enable-lua-records` to ``no``, this will disable :ref:`Lua Records`. + +Depending on your specific requirements, it might be good perform checks on zone files before loading the zone into PowerDNS to: + +- Enforce reasonable ``TTL`` values. +- Enforce reasonable values in the ``SOA`` records. +- Validate delegations. +- Enforce a reasonable maximum for the total number of records. +- Enforce a reasonable maximum for the number of records per record set. diff --git a/docs/settings.rst b/docs/settings.rst index 693ee3061c9c..9060dfe4d770 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -134,7 +134,10 @@ Enable/disable the :doc:`http-api/index`. - String -Static pre-shared authentication key for access to the REST API. +.. versionchanged:: 4.6.0 + This setting now accepts a hashed and salted version. + +Static pre-shared authentication key for access to the REST API. Since 4.6.0 the key can be hashed and salted using ``pdnsutil hash-password`` instead of being stored in the configuration in plaintext, but the plaintext version is still supported. .. _setting-autosecondary: @@ -250,6 +253,8 @@ You may specify an alternate port by appending :port, ex: - Path If set, chroot to this directory for more security. See :doc:`security`. +This is not recommended; instead, we recommend containing PowerDNS using operating system features. +We ship systemd unit files with our packages to make this easy. Make sure that ``/dev/log`` is available from within the chroot. Logging will silently fail over time otherwise (on logrotate). @@ -260,9 +265,9 @@ set in the configuration are relative to the new root. When running on a system where systemd manages services, ``chroot`` does not work out of the box, as PowerDNS cannot use the ``NOTIFY_SOCKET``. -Either don't ``chroot`` on these systems or set the 'Type' of the this +Either don't ``chroot`` on these systems or set the 'Type' of the service to 'simple' instead of 'notify' (refer to the systemd -documentation on how to modify unit-files) +documentation on how to modify unit-files). .. _setting-config-dir: @@ -619,6 +624,21 @@ ADDITIONAL section when sending a referral. Seconds to cache zone metadata from the database. A value of 0 disables caching. +.. _setting-edns-cookie-secret: + +``edns-cookie-secret`` +-------------------------- + +.. versionadded:: 4.6.0 + +- String +- Default: (empty) + +When set, PowerDNS will respond with :rfc:`9018` EDNS Cookies to queries that have the EDNS0 Cookie option. +PowerDNS will also respond with BADCOOKIE to clients that have sent only a client cookie, or a bad server cookie (section 5.2.3 and 5.2.4 of :rfc:`7873`). + +This setting MUST be 32 hexadecimal characters, as the siphash algorithm's key used to create the cookie requires a 128-bit key. + .. _setting-edns-subnet-processing: ``edns-subnet-processing`` @@ -993,6 +1013,17 @@ will generally suffice for most installations. Maximum number of empty non-terminals to add to a zone. This is a protection measure to avoid database explosion due to long names. +.. _setting-max-include-depth: + +``max-include-depth`` +---------------------- + +- Integer +- Default: 20 + +Maximum number of nested ``$INCLUDE`` directives while processing a zone file. +Zero mean no ``$INCLUDE`` directives will be accepted. + .. _setting-max-generate-steps: ``max-generate-steps`` @@ -1253,6 +1284,34 @@ prevent-self-notification to "no". Turn on operating as a primary. See :ref:`primary-operation`. +.. _setting-proxy-protocol-from: + +``proxy-protocol-from`` +----------------------- +.. versionadded:: 4.6.0 + +- IP addresses or netmasks, separated by commas +- Default: empty + +Ranges that are required to send a Proxy Protocol version 2 header in front of UDP and TCP queries, to pass the original source and destination addresses and ports to the Authoritative. +Queries that are not prefixed with such a header will not be accepted from clients in these ranges. Queries prefixed by headers from clients that are not listed in these ranges will be dropped. + +Note that once a Proxy Protocol header has been received, the source address from the proxy header instead of the address of the proxy will be checked against primary addresses sending NOTIFYs, and the ACLs for any client requesting AXFRs. +When using this setting combined with :ref:`setting-trusted-notification-proxy`, please be aware that the trusted address will also be checked against the source address in the PROXY header. + +The dnsdist docs have `more information about the PROXY protocol `_. + +.. _setting-proxy-protocol-maximum-size: + +``proxy-protocol-maximum-size`` +------------------------------- +.. versionadded:: 4.6.0 + +- Integer +- Default: 512 + +The maximum size, in bytes, of a Proxy Protocol payload (header, addresses and ports, and TLV values). Queries with a larger payload will be dropped. + .. _setting-query-cache-ttl: ``query-cache-ttl`` @@ -1771,6 +1830,18 @@ IP Address for webserver/API to listen on. Webserver/API access is only allowed from these subnets. +.. _setting-webserver-hash-plaintext-credentials: + +``webserver-hash-plaintext-credentials`` +---------------------------------------- +..versionadded:: 4.6.0 + +- Boolean +- Default: no + +Whether passwords and API keys supplied in the configuration as plaintext should be hashed during startup, to prevent the plaintext versions from staying in memory. Doing so increases significantly the cost of verifying credentials and is thus disabled by default. +Note that this option only applies to credentials stored in the configuration as plaintext, but hashed credentials are supported without enabling this option. + .. _setting-webserver-loglevel: ``webserver-loglevel`` @@ -1825,10 +1896,12 @@ Maximum request/response body size in megabytes. ``webserver-password`` ---------------------- +.. versionchanged:: 4.6.0 + This setting now accepts a hashed and salted version. - String -The plaintext password required for accessing the webserver. +Password required to access the webserver. Since 4.6.0 the password can be hashed and salted using ``pdnsutil hash-password`` instead of being present in the configuration in plaintext, but the plaintext version is still supported. .. _setting-webserver-port: diff --git a/docs/upgrading.rst b/docs/upgrading.rst index 51e630790746..93e31abb6205 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -8,9 +8,30 @@ Please upgrade to the PowerDNS Authoritative Server 4.0.0 from 3.4.2+. See the `3.X `__ upgrade notes if your version is older than 3.4.2. -4.4.x to 4.5.0 or master +4.5.x to 4.6.0 or master ------------------------ +Automatic conversion of ``@`` signs in SOA +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Before version 4.5.0, PowerDNS would automatically replace ``@`` in the SOA RNAME with ``.``, making it easy for users to enter their hostmaster email address without having to think about syntax. +However, this feature interacts badly with handling of presigned zones. +In version 4.5.0, this feature was accidentally broken in the implementation of the zone cache. +In 4.6.0, this automatic conversion is fully removed. +If you still have ``@`` signs in any SOA RNAMEs, 4.6.0 will serve those out literally. +You can find any stray ``@`` signs by running ``pdnsutil check-all-zones``. + +4.4.x to 4.5.0 +-------------- + +Automatic conversion of ``@`` signs in SOA +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Before version 4.5.0, PowerDNS would automatically replace ``@`` in the SOA RNAME with ``.``, making it easy for users to enter their hostmaster email address without having to think about syntax. +In version 4.5.0, this feature was accidentally broken in the implementation of the zone cache, and the replacement would only happen if the zone cache was disabled. +Note that in 4.6.0, this automatic conversion is fully removed. +If you still have ``@`` signs in any SOA RNAMEs, 4.5.0 will serve those out literally if the zone cache is enabled. + Record type changes ^^^^^^^^^^^^^^^^^^^ diff --git a/ext/lmdb-safe/lmdb-safe.cc b/ext/lmdb-safe/lmdb-safe.cc index 7b968e20d655..7a023fd05afc 100644 --- a/ext/lmdb-safe/lmdb-safe.cc +++ b/ext/lmdb-safe/lmdb-safe.cc @@ -176,18 +176,9 @@ MDB_txn *MDBRWTransactionImpl::openRWTransaction(MDBEnv *env, MDB_txn *parent, i if(env->getROTX() || env->getRWTX()) throw std::runtime_error("Duplicate RW transaction"); - for(int tries =0 ; tries < 3; ++tries) { // it might happen twice, who knows - if(int rc=mdb_txn_begin(env->d_env, parent, flags, &result)) { - if(rc == MDB_MAP_RESIZED && tries < 2) { - // "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED. - // call mdb_env_set_mapsize with a size of zero to adopt the new size." - mdb_env_set_mapsize(env->d_env, 0); - continue; - } - throw std::runtime_error("Unable to start RW transaction: "+std::string(mdb_strerror(rc))); - } - break; - } + if(int rc=mdb_txn_begin(env->d_env, parent, flags, &result)) + throw std::runtime_error("Unable to start RW transaction: "+std::string(mdb_strerror(rc))); + env->incRWTX(); return result; } @@ -245,19 +236,10 @@ MDB_txn *MDBROTransactionImpl::openROTransaction(MDBEnv *env, MDB_txn *parent, i /* A transaction and its cursors must only be used by a single thread, and a thread may only have a single transaction at a time. If MDB_NOTLS is in use, this does not apply to read-only transactions. */ MDB_txn *result = nullptr; - for(int tries =0 ; tries < 3; ++tries) { // it might happen twice, who knows - if(int rc=mdb_txn_begin(env->d_env, parent, MDB_RDONLY | flags, &result)) { - if(rc == MDB_MAP_RESIZED && tries < 2) { - // "If the mapsize is increased by another process (..) mdb_txn_begin() will return MDB_MAP_RESIZED. - // call mdb_env_set_mapsize with a size of zero to adopt the new size." - mdb_env_set_mapsize(env->d_env, 0); - continue; - } - - throw std::runtime_error("Unable to start RO transaction: "+string(mdb_strerror(rc))); - } - break; - } + + if(int rc=mdb_txn_begin(env->d_env, parent, MDB_RDONLY | flags, &result)) + throw std::runtime_error("Unable to start RO transaction: "+string(mdb_strerror(rc))); + env->incROTX(); return result; diff --git a/m4/pdns_check_libcrypto.m4 b/m4/pdns_check_libcrypto.m4 index 789f29416432..8e1219a3e75b 100644 --- a/m4/pdns_check_libcrypto.m4 +++ b/m4/pdns_check_libcrypto.m4 @@ -112,6 +112,7 @@ AC_DEFUN([PDNS_CHECK_LIBCRYPTO], [ [ AC_MSG_RESULT([yes]) AC_CHECK_FUNCS([RAND_bytes RAND_pseudo_bytes CRYPTO_memcmp OPENSSL_init_crypto EVP_MD_CTX_new EVP_MD_CTX_free RSA_get0_key]) + AC_CHECK_DECL(EVP_PKEY_CTX_set1_scrypt_salt, [AC_DEFINE([HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT], [1], [Define to 1 if you have EVP_PKEY_CTX_set1_scrypt_salt])], [], [#include ]) $1 ], [ AC_MSG_RESULT([no]) diff --git a/m4/pdns_with_gnutls.m4 b/m4/pdns_with_gnutls.m4 index c693dff81fdb..b6ad100bbbd6 100644 --- a/m4/pdns_with_gnutls.m4 +++ b/m4/pdns_with_gnutls.m4 @@ -18,7 +18,7 @@ AC_DEFUN([PDNS_WITH_GNUTLS], [ save_LIBS=$LIBS CFLAGS="$GNUTLS_CFLAGS $CFLAGS" LIBS="$GNUTLS_LIBS $LIBS" - AC_CHECK_FUNCS([gnutls_memset gnutls_session_set_verify_cert gnutls_session_get_verify_cert_status]) + AC_CHECK_FUNCS([gnutls_memset gnutls_session_set_verify_cert gnutls_session_get_verify_cert_status gnutls_alpn_set_protocols]) CFLAGS=$save_CFLAGS LIBS=$save_LIBS diff --git a/m4/pdns_with_libsodium.m4 b/m4/pdns_with_libsodium.m4 index 44e034b2d267..fbe60d29231a 100644 --- a/m4/pdns_with_libsodium.m4 +++ b/m4/pdns_with_libsodium.m4 @@ -15,7 +15,7 @@ AC_DEFUN([PDNS_WITH_LIBSODIUM], [ save_LIBS=$LIBS CFLAGS="$LIBSODIUM_CFLAGS $CFLAGS" LIBS="$LIBSODIUM_LIBS $LIBS" - AC_CHECK_FUNCS([crypto_box_easy_afternm crypto_box_curve25519xchacha20poly1305_easy randombytes_stir]) + AC_CHECK_FUNCS([crypto_box_easy_afternm crypto_box_curve25519xchacha20poly1305_easy randombytes_stir sodium_memcmp crypto_shorthash]) CFLAGS=$save_CFLAGS LIBS=$save_LIBS ], [ : ]) diff --git a/m4/pdns_with_libssl.m4 b/m4/pdns_with_libssl.m4 index c42905fd1dba..3e32bc40864f 100644 --- a/m4/pdns_with_libssl.m4 +++ b/m4/pdns_with_libssl.m4 @@ -17,7 +17,7 @@ AC_DEFUN([PDNS_WITH_LIBSSL], [ save_LIBS=$LIBS CFLAGS="$LIBSSL_CFLAGS $CFLAGS" LIBS="$LIBSSL_LIBS -lcrypto $LIBS" - AC_CHECK_FUNCS([SSL_CTX_set_ciphersuites OCSP_basic_sign SSL_CTX_set_num_tickets SSL_CTX_set_keylog_callback SSL_CTX_get0_privatekey SSL_CTX_set_min_proto_version SSL_set_hostflags]) + AC_CHECK_FUNCS([SSL_CTX_set_ciphersuites OCSP_basic_sign SSL_CTX_set_num_tickets SSL_CTX_set_keylog_callback SSL_CTX_get0_privatekey SSL_CTX_set_min_proto_version SSL_set_hostflags SSL_CTX_set_alpn_protos SSL_CTX_set_next_proto_select_cb SSL_get0_alpn_selected SSL_get0_next_proto_negotiated SSL_CTX_set_alpn_select_cb]) CFLAGS=$save_CFLAGS LIBS=$save_LIBS diff --git a/m4/systemd.m4 b/m4/systemd.m4 index 3c8f19678373..66dc62cdeb80 100644 --- a/m4/systemd.m4 +++ b/m4/systemd.m4 @@ -174,6 +174,9 @@ AC_DEFUN([AX_CHECK_SYSTEMD_FEATURES], [ # @aio, @sync, @chown, @setuid, @memlock, @signal and @timer in 235 systemd_system_call_filter=y fi + if test $_systemd_version -ge 236; then + systemd_percent_t=y + fi if test $_systemd_version -ge 239; then systemd_private_mounts=y fi @@ -195,6 +198,7 @@ AC_DEFUN([AX_CHECK_SYSTEMD_FEATURES], [ AM_CONDITIONAL([HAVE_SYSTEMD_DYNAMIC_USER], [ test x"$systemd_dynamic_user" = "xy" ]) AM_CONDITIONAL([HAVE_SYSTEMD_LOCK_PERSONALITY], [ test x"$systemd_lock_personality" = "xy" ]) AM_CONDITIONAL([HAVE_SYSTEMD_MEMORY_DENY_WRITE_EXECUTE], [ test x"$systemd_memory_deny_write_execute" = "xy" ]) + AM_CONDITIONAL([HAVE_SYSTEMD_PERCENT_T], [ test x"$systemd_percent_t" = "xy" ]) AM_CONDITIONAL([HAVE_SYSTEMD_PRIVATE_DEVICES], [ test x"$systemd_private_devices" = "xy" ]) AM_CONDITIONAL([HAVE_SYSTEMD_PRIVATE_MOUNTS], [ test x"$systemd_private_mounts" = "xy" ]) AM_CONDITIONAL([HAVE_SYSTEMD_PRIVATE_TMP], [ test x"$systemd_private_tmp" = "xy" ]) diff --git a/modules/Makefile.am b/modules/Makefile.am index bc4782ccadb1..1c51fb643885 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -11,6 +11,5 @@ DIST_SUBDIRS = \ lmdbbackend \ lua2backend \ pipebackend \ - randombackend \ remotebackend \ tinydnsbackend diff --git a/modules/bindbackend/bindbackend2.cc b/modules/bindbackend/bindbackend2.cc index 669d78e99faf..2922798a9ed5 100644 --- a/modules/bindbackend/bindbackend2.cc +++ b/modules/bindbackend/bindbackend2.cc @@ -66,7 +66,7 @@ or safeRemoveBBDomainInfo. These all lock as they should. Several functions need to traverse s_state to get data for the rest of PowerDNS. When doing so, - you need to manually take the s_state_lock (read). + you need to manually take the lock (read). Parsing zones happens with parseZone(), which fills a BB2DomainInfo object. This can then be stored with safePutBBDomainInfo. @@ -74,18 +74,14 @@ the records might be in use in other places. */ -Bind2Backend::state_t Bind2Backend::s_state; +SharedLockGuarded Bind2Backend::s_state; int Bind2Backend::s_first = 1; bool Bind2Backend::s_ignore_broken_records = false; -ReadWriteLock Bind2Backend::s_state_lock; std::mutex Bind2Backend::s_supermaster_config_lock; // protects writes to config file std::mutex Bind2Backend::s_startup_lock; string Bind2Backend::s_binddirectory; -template -std::mutex LookButDontTouch::s_lock; - BB2DomainInfo::BB2DomainInfo() { d_loaded = false; @@ -137,44 +133,45 @@ void BB2DomainInfo::setCtime() bool Bind2Backend::safeGetBBDomainInfo(int id, BB2DomainInfo* bbd) { - ReadLock rl(&s_state_lock); - state_t::const_iterator iter = s_state.find(id); - if (iter == s_state.end()) + auto state = s_state.read_lock(); + state_t::const_iterator iter = state->find(id); + if (iter == state->end()) { return false; + } *bbd = *iter; return true; } bool Bind2Backend::safeGetBBDomainInfo(const DNSName& name, BB2DomainInfo* bbd) { - ReadLock rl(&s_state_lock); - typedef state_t::index::type nameindex_t; - nameindex_t& nameindex = boost::multi_index::get(s_state); - - nameindex_t::const_iterator iter = nameindex.find(name); - if (iter == nameindex.end()) + auto state = s_state.read_lock(); + auto& nameindex = boost::multi_index::get(*state); + auto iter = nameindex.find(name); + if (iter == nameindex.end()) { return false; + } *bbd = *iter; return true; } bool Bind2Backend::safeRemoveBBDomainInfo(const DNSName& name) { - WriteLock rl(&s_state_lock); + auto state = s_state.write_lock(); typedef state_t::index::type nameindex_t; - nameindex_t& nameindex = boost::multi_index::get(s_state); + nameindex_t& nameindex = boost::multi_index::get(*state); nameindex_t::iterator iter = nameindex.find(name); - if (iter == nameindex.end()) + if (iter == nameindex.end()) { return false; + } nameindex.erase(iter); return true; } void Bind2Backend::safePutBBDomainInfo(const BB2DomainInfo& bbd) { - WriteLock rl(&s_state_lock); - replacing_insert(s_state, bbd); + auto state = s_state.write_lock(); + replacing_insert(*state, bbd); } void Bind2Backend::setNotified(uint32_t id, uint32_t serial) @@ -227,7 +224,7 @@ bool Bind2Backend::startTransaction(const DNSName& qname, int id) return false; } - d_of = std::unique_ptr(new ofstream(d_transaction_tmpname.c_str())); + d_of = std::make_unique(d_transaction_tmpname); if (!*d_of) { unlink(d_transaction_tmpname.c_str()); close(fd); @@ -326,9 +323,9 @@ void Bind2Backend::getUpdatedMasters(vector* changedDomains) { vector consider; { - ReadLock rl(&s_state_lock); + auto state = s_state.read_lock(); - for (const auto& i : s_state) { + for (const auto& i : *state) { if (i.d_kind != DomainInfo::Master && this->alsoNotify.empty() && i.d_also_notify.empty()) continue; @@ -366,16 +363,16 @@ void Bind2Backend::getUpdatedMasters(vector* changedDomains) } } -void Bind2Backend::getAllDomains(vector* domains, bool include_disabled) +void Bind2Backend::getAllDomains(vector* domains, bool getSerial, bool include_disabled) { SOAData soadata; // prevent deadlock by using getSOA() later on { - ReadLock rl(&s_state_lock); - domains->reserve(s_state.size()); + auto state = s_state.read_lock(); + domains->reserve(state->size()); - for (const auto& i : s_state) { + for (const auto& i : *state) { DomainInfo di; di.id = i.d_id; di.zone = i.d_name; @@ -387,17 +384,19 @@ void Bind2Backend::getAllDomains(vector* domains, bool include_disab }; } - for (DomainInfo& di : *domains) { - // do not corrupt di if domain supplied by another backend. - if (di.backend != this) - continue; - try { - this->getSOA(di.zone, soadata); - } - catch (...) { - continue; + if (getSerial) { + for (DomainInfo& di : *domains) { + // do not corrupt di if domain supplied by another backend. + if (di.backend != this) + continue; + try { + this->getSOA(di.zone, soadata); + } + catch (...) { + continue; + } + di.serial = soadata.serial; } - di.serial = soadata.serial; } } @@ -405,9 +404,9 @@ void Bind2Backend::getUnfreshSlaveInfos(vector* unfreshDomains) { vector domains; { - ReadLock rl(&s_state_lock); - domains.reserve(s_state.size()); - for (const auto& i : s_state) { + auto state = s_state.read_lock(); + domains.reserve(state->size()); + for (const auto& i : *state) { if (i.d_kind != DomainInfo::Slave) continue; DomainInfo sd; @@ -478,8 +477,8 @@ void Bind2Backend::alsoNotifies(const DNSName& domain, set* ips) (*ips).insert(str); } } - ReadLock rl(&s_state_lock); - for (const auto& i : s_state) { + auto state = s_state.read_lock(); + for (const auto& i : *state) { if (i.d_name == domain) { for (const auto& it : i.d_also_notify) { (*ips).insert(it); @@ -504,6 +503,7 @@ void Bind2Backend::parseZoneFile(BB2DomainInfo* bbd) auto records = std::make_shared(); ZoneParserTNG zpt(bbd->d_filename, bbd->d_name, s_binddirectory, d_upgradeContent); zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps")); + zpt.setMaxIncludes(::arg().asNum("max-include-depth")); DNSResourceRecord rr; string hashed; while (zpt.get(rr)) { @@ -518,7 +518,7 @@ void Bind2Backend::parseZoneFile(BB2DomainInfo* bbd) bbd->d_loaded = true; bbd->d_checknow = false; bbd->d_status = "parsed into memory at " + nowTime(); - bbd->d_records = LookButDontTouch(records); + bbd->d_records = LookButDontTouch(std::move(records)); bbd->d_nsec3zone = nsec3zone; bbd->d_nsec3param = ns3pr; } @@ -604,8 +604,8 @@ string Bind2Backend::DLDomStatusHandler(const vector& parts, Utility::pi } } else { - ReadLock rl(&s_state_lock); - for (const auto& i : s_state) { + auto state = s_state.read_lock(); + for (const auto& i : *state) { ret << i.d_name << ": " << (i.d_loaded ? "" : "[rejected]") << "\t" << i.d_status << "\n"; } } @@ -666,8 +666,8 @@ string Bind2Backend::DLDomExtendedStatusHandler(const vector& parts, Uti } } else { - ReadLock rl(&s_state_lock); - for (const auto& state : s_state) { + auto rstate = s_state.read_lock(); + for (const auto& state : *rstate) { printDomainExtendedStatus(ret, state); } } @@ -682,8 +682,8 @@ string Bind2Backend::DLDomExtendedStatusHandler(const vector& parts, Uti string Bind2Backend::DLListRejectsHandler(const vector& parts, Utility::pid_t ppid) { ostringstream ret; - ReadLock rl(&s_state_lock); - for (const auto& i : s_state) { + auto rstate = s_state.read_lock(); + for (const auto& i : *rstate) { if (!i.d_loaded) ret << i.d_name << "\t" << i.d_status << endl; } @@ -783,8 +783,8 @@ void Bind2Backend::rediscover(string* status) void Bind2Backend::reload() { - WriteLock rwl(&s_state_lock); - for (const auto& i : s_state) { + auto state = s_state.write_lock(); + for (const auto& i : *state) { i.d_checknow = true; // being a bit cheeky here, don't index state_t on this (mutable) } } @@ -903,8 +903,8 @@ void Bind2Backend::loadConfig(string* status) set oldnames, newnames; { - ReadLock rl(&s_state_lock); - for (const BB2DomainInfo& bbd : s_state) { + auto state = s_state.read_lock(); + for (const BB2DomainInfo& bbd : *state) { oldnames.insert(bbd.d_name); } } @@ -1041,12 +1041,12 @@ void Bind2Backend::queueReloadAndStore(unsigned int id) try { if (!safeGetBBDomainInfo(id, &bbold)) return; + bbold.d_checknow = false; BB2DomainInfo bbnew(bbold); /* make sure that nothing will be able to alter the existing records, we will load them from the zone file instead */ bbnew.d_records = LookButDontTouch(); parseZoneFile(&bbnew); - bbnew.d_checknow = false; bbnew.d_wasRejectedLastReload = false; safePutBBDomainInfo(bbnew); g_log << Logger::Warning << "Zone '" << bbnew.d_name << "' (" << bbnew.d_filename << ") reloaded" << endl; @@ -1054,16 +1054,18 @@ void Bind2Backend::queueReloadAndStore(unsigned int id) catch (PDNSException& ae) { ostringstream msg; msg << " error at " + nowTime() + " parsing '" << bbold.d_name << "' from file '" << bbold.d_filename << "': " << ae.reason; - g_log << Logger::Warning << " error parsing '" << bbold.d_name << "' from file '" << bbold.d_filename << "': " << ae.reason << endl; + g_log << Logger::Warning << "Error parsing '" << bbold.d_name << "' from file '" << bbold.d_filename << "': " << ae.reason << endl; bbold.d_status = msg.str(); + bbold.d_lastcheck = time(nullptr); bbold.d_wasRejectedLastReload = true; safePutBBDomainInfo(bbold); } catch (std::exception& ae) { ostringstream msg; msg << " error at " + nowTime() + " parsing '" << bbold.d_name << "' from file '" << bbold.d_filename << "': " << ae.what(); - g_log << Logger::Warning << " error parsing '" << bbold.d_name << "' from file '" << bbold.d_filename << "': " << ae.what() << endl; + g_log << Logger::Warning << "Error parsing '" << bbold.d_name << "' from file '" << bbold.d_filename << "': " << ae.what() << endl; bbold.d_status = msg.str(); + bbold.d_lastcheck = time(nullptr); bbold.d_wasRejectedLastReload = true; safePutBBDomainInfo(bbold); } @@ -1179,11 +1181,6 @@ void Bind2Backend::lookup(const QType& qtype, const DNSName& qname, int zoneId, d_handle.qtype = qtype; d_handle.domain = std::move(domain); - if (!bbd.d_loaded) { - d_handle.reset(); - throw DBException("Zone for '" + d_handle.domain.toLogString() + "' in '" + bbd.d_filename + "' temporarily not available (file missing, or master dead)"); // fsck - } - if (!bbd.current()) { g_log << Logger::Warning << "Zone '" << d_handle.domain << "' (" << bbd.d_filename << ") needs reloading" << endl; queueReloadAndStore(bbd.d_id); @@ -1191,6 +1188,11 @@ void Bind2Backend::lookup(const QType& qtype, const DNSName& qname, int zoneId, throw DBException("Zone '" + bbd.d_name.toLogString() + "' (" + bbd.d_filename + ") gone after reload"); // if we don't throw here, we crash for some reason } + if (!bbd.d_loaded) { + d_handle.reset(); + throw DBException("Zone for '" + d_handle.domain.toLogString() + "' in '" + bbd.d_filename + "' not loaded (file missing, corrupt or master dead)"); // fsck + } + d_handle.d_records = bbd.d_records.get(); if (d_handle.d_records->empty()) @@ -1299,6 +1301,10 @@ bool Bind2Backend::list(const DNSName& target, int id, bool include_disabled) d_handle.reset(); DLOG(g_log << "Bind2Backend constructing handle for list of " << id << endl); + if (!bbd.d_loaded) { + throw PDNSException("zone was not loaded, perhaps because of: " + bbd.d_status); + } + d_handle.d_records = bbd.d_records.get(); // give it a copy, which will stay around d_handle.d_qname_iter = d_handle.d_records->begin(); d_handle.d_qname_end = d_handle.d_records->end(); // iter now points to a vector of pointers to vector @@ -1364,11 +1370,11 @@ BB2DomainInfo Bind2Backend::createDomainEntry(const DNSName& domain, const strin { int newid = 1; { // Find a free zone id nr. - ReadLock rl(&s_state_lock); - if (!s_state.empty()) { + auto state = s_state.read_lock(); + if (!state->empty()) { // older (1.53) versions of boost have an expression for s_state.rbegin() // that is ambiguous in C++17. So construct it explicitly - newid = boost::make_reverse_iterator(s_state.end())->d_id + 1; + newid = boost::make_reverse_iterator(state->end())->d_id + 1; } } @@ -1427,14 +1433,18 @@ bool Bind2Backend::searchRecords(const string& pattern, int maxResults, vector rhandle = h.d_records.get(); for (recordstorage_t::const_iterator ri = rhandle->begin(); result.size() < static_cast::size_type>(maxResults) && ri != rhandle->end(); ri++) { diff --git a/modules/bindbackend/bindbackend2.hh b/modules/bindbackend/bindbackend2.hh index fb4ed0467265..db65ff266fee 100644 --- a/modules/bindbackend/bindbackend2.hh +++ b/modules/bindbackend/bindbackend2.hh @@ -107,36 +107,34 @@ typedef multi_index_container< recordstorage_t; template -class LookButDontTouch // : public boost::noncopyable +class LookButDontTouch { public: LookButDontTouch() { } - LookButDontTouch(shared_ptr records) : - d_records(records) + LookButDontTouch(shared_ptr&& records) : + d_records(std::move(records)) { } shared_ptr get() { - shared_ptr ret; - { - std::lock_guard lock(s_lock); - ret = d_records; - } - return ret; + return d_records; } size_t getEntriesCount() const { - std::lock_guard lock(s_lock); + if (!d_records) { + return 0; + } return d_records->size(); } private: - static std::mutex s_lock; - shared_ptr d_records; + /* we can increase the number of references to that object, + but never update the object itself */ + shared_ptr d_records; }; /** Class which describes all metadata of a domain for storage by the Bind2Backend, and also contains a pointer to a vector of Bind2DNSRecord's */ @@ -196,7 +194,7 @@ public: void lookup(const QType&, const DNSName& qdomain, int zoneId, DNSPacket* p = nullptr) override; bool list(const DNSName& target, int id, bool include_disabled = false) override; bool get(DNSResourceRecord&) override; - void getAllDomains(vector* domains, bool include_disabled = false) override; + void getAllDomains(vector* domains, bool getSerial, bool include_disabled = false) override; static DNSBackend* maker(); static std::mutex s_startup_lock; @@ -233,8 +231,7 @@ public: indexed_by>, ordered_unique, member>>> state_t; - static state_t s_state; - static ReadWriteLock s_state_lock; + static SharedLockGuarded s_state; void parseZoneFile(BB2DomainInfo* bbd); void rediscover(string* status = nullptr) override; diff --git a/modules/geoipbackend/geoipbackend.cc b/modules/geoipbackend/geoipbackend.cc index 3d563eab218b..e1a2312a6c42 100644 --- a/modules/geoipbackend/geoipbackend.cc +++ b/modules/geoipbackend/geoipbackend.cc @@ -850,7 +850,7 @@ bool GeoIPBackend::getDomainInfo(const DNSName& domain, DomainInfo& di, bool get return false; } -void GeoIPBackend::getAllDomains(vector* domains, bool include_disabled) +void GeoIPBackend::getAllDomains(vector* domains, bool getSerial, bool include_disabled) { ReadLock rl(&s_state_lock); diff --git a/modules/geoipbackend/geoipbackend.hh b/modules/geoipbackend/geoipbackend.hh index cb463cc08010..470da9bdb03e 100644 --- a/modules/geoipbackend/geoipbackend.hh +++ b/modules/geoipbackend/geoipbackend.hh @@ -55,7 +55,7 @@ public: void reload() override; void rediscover(string* status = 0) override; bool getDomainInfo(const DNSName& domain, DomainInfo& di, bool getSerial = true) override; - void getAllDomains(vector* domains, bool include_disabled = false) override; + void getAllDomains(vector* domains, bool getSerial, bool include_disabled) override; // dnssec support bool doesDNSSEC() override { return d_dnssec; }; diff --git a/modules/geoipbackend/geoipinterface-dat.cc b/modules/geoipbackend/geoipinterface-dat.cc index 5a5a05dac4fb..75b11ccc58f8 100644 --- a/modules/geoipbackend/geoipinterface-dat.cc +++ b/modules/geoipbackend/geoipinterface-dat.cc @@ -476,7 +476,7 @@ unique_ptr GeoIPInterface::makeDATInterface(const string& fname, const auto& opt = opts.find("mode"); if (opt != opts.end()) mode = opt->second; - return unique_ptr(new GeoIPInterfaceDAT(fname, mode)); + return std::make_unique(fname, mode); } #else diff --git a/modules/geoipbackend/geoipinterface-mmdb.cc b/modules/geoipbackend/geoipinterface-mmdb.cc index b8a11ef806a9..969d5abc5293 100644 --- a/modules/geoipbackend/geoipinterface-mmdb.cc +++ b/modules/geoipbackend/geoipinterface-mmdb.cc @@ -283,7 +283,7 @@ unique_ptr GeoIPInterface::makeMMDBInterface(const string& fname const auto& opt_lang = opts.find("language"); if (opt_lang != opts.end()) language = opt_lang->second; - return unique_ptr(new GeoIPInterfaceMMDB(fname, mode, language)); + return std::make_unique(fname, mode, language); } #else diff --git a/modules/gmysqlbackend/gmysqlbackend.cc b/modules/gmysqlbackend/gmysqlbackend.cc index a7da009a68c0..97c010ebddd1 100644 --- a/modules/gmysqlbackend/gmysqlbackend.cc +++ b/modules/gmysqlbackend/gmysqlbackend.cc @@ -128,7 +128,7 @@ class gMySQLFactory : public BackendFactory declare(suffix, "update-account-query", "", "update domains set account=? where name=?"); declare(suffix, "update-serial-query", "", "update domains set notified_serial=? where id=?"); declare(suffix, "update-lastcheck-query", "", "update domains set last_check=? where id=?"); - declare(suffix, "info-all-master-query", "", "select d.id, d.name, d.notified_serial, r.content from records r join domains d on r.name=d.name where r.type='SOA' and r.disabled=0 and d.type='MASTER'"); + declare(suffix, "info-all-master-query", "", "select d.id, d.name, d.notified_serial, r.content from records r join domains d on r.domain_id=d.id and r.name=d.name where r.type='SOA' and r.disabled=0 and d.type='MASTER'"); declare(suffix, "delete-domain-query", "", "delete from domains where name=?"); declare(suffix, "delete-zone-query", "", "delete from records where domain_id=?"); declare(suffix, "delete-rrset-query", "", "delete from records where domain_id=? and name=? and type=?"); diff --git a/modules/gmysqlbackend/smysql.cc b/modules/gmysqlbackend/smysql.cc index 35267c9e58f0..8b3c80bbc556 100644 --- a/modules/gmysqlbackend/smysql.cc +++ b/modules/gmysqlbackend/smysql.cc @@ -476,13 +476,16 @@ void SMySQL::connect() { int retry = 1; - std::lock_guard l(s_myinitlock); - if (d_threadCleanup) { - threadcloser.enable(); - } + { + std::lock_guard l(s_myinitlock); + if (d_threadCleanup) { + threadcloser.enable(); + } - if (!mysql_init(&d_db)) - throw sPerrorException("Unable to initialize mysql driver"); + if (!mysql_init(&d_db)) { + throw sPerrorException("Unable to initialize mysql driver"); + } + } do { @@ -549,7 +552,7 @@ SSqlException SMySQL::sPerrorException(const string& reason) std::unique_ptr SMySQL::prepare(const string& query, int nparams) { - return std::unique_ptr(new SMySQLStatement(query, s_dolog, nparams, &d_db)); + return std::make_unique(query, s_dolog, nparams, &d_db); } void SMySQL::execute(const string& query) diff --git a/modules/godbcbackend/godbcbackend.cc b/modules/godbcbackend/godbcbackend.cc index b41bdc3509ba..ee28a581b657 100644 --- a/modules/godbcbackend/godbcbackend.cc +++ b/modules/godbcbackend/godbcbackend.cc @@ -108,7 +108,7 @@ class gODBCFactory : public BackendFactory declare(suffix, "update-account-query", "", "update domains set account=? where name=?"); declare(suffix, "update-serial-query", "", "update domains set notified_serial=? where id=?"); declare(suffix, "update-lastcheck-query", "", "update domains set last_check=? where id=?"); - declare(suffix, "info-all-master-query", "", "select domains.id, domains.name, domains.notified_serial, records.content from records join domains on records.name=domains.name where records.type='SOA' and records.disabled=0 and domains.type='MASTER'"); + declare(suffix, "info-all-master-query", "", "select domains.id, domains.name, domains.notified_serial, records.content from records join domains on records.domain_id=domains.id and records.name=domains.name where records.type='SOA' and records.disabled=0 and domains.type='MASTER'"); declare(suffix, "delete-domain-query", "", "delete from domains where name=?"); declare(suffix, "delete-zone-query", "", "delete from records where domain_id=?"); declare(suffix, "delete-rrset-query", "", "delete from records where domain_id=? and name=? and type=?"); diff --git a/modules/godbcbackend/sodbc.cc b/modules/godbcbackend/sodbc.cc index 2985557a770f..a1a7ad0255bf 100644 --- a/modules/godbcbackend/sodbc.cc +++ b/modules/godbcbackend/sodbc.cc @@ -479,7 +479,7 @@ SSqlException SODBC::sPerrorException(const std::string& reason) std::unique_ptr SODBC::prepare(const string& query, int nparams) { - return std::unique_ptr(new SODBCStatement(query, m_log, nparams, m_connection)); + return std::make_unique(query, m_log, nparams, m_connection); } void SODBC::startTransaction() diff --git a/modules/gpgsqlbackend/gpgsqlbackend.cc b/modules/gpgsqlbackend/gpgsqlbackend.cc index 500bab200aab..61563740221b 100644 --- a/modules/gpgsqlbackend/gpgsqlbackend.cc +++ b/modules/gpgsqlbackend/gpgsqlbackend.cc @@ -135,7 +135,7 @@ class gPgSQLFactory : public BackendFactory declare(suffix, "update-account-query", "", "update domains set account=$1 where name=$2"); declare(suffix, "update-serial-query", "", "update domains set notified_serial=$1 where id=$2"); declare(suffix, "update-lastcheck-query", "", "update domains set last_check=$1 where id=$2"); - declare(suffix, "info-all-master-query", "", "select domains.id, domains.name, domains.notified_serial, records.content from records join domains on records.name=domains.name where records.type='SOA' and records.disabled=false and domains.type='MASTER'"); + declare(suffix, "info-all-master-query", "", "select domains.id, domains.name, domains.notified_serial, records.content from records join domains on records.domain_id=domains.id and records.name=domains.name where records.type='SOA' and records.disabled=false and domains.type='MASTER'"); declare(suffix, "delete-domain-query", "", "delete from domains where name=$1"); declare(suffix, "delete-zone-query", "", "delete from records where domain_id=$1"); declare(suffix, "delete-rrset-query", "", "delete from records where domain_id=$1 and name=$2 and type=$3"); diff --git a/modules/gpgsqlbackend/spgsql.cc b/modules/gpgsqlbackend/spgsql.cc index c40347fd6768..996d72dbc7aa 100644 --- a/modules/gpgsqlbackend/spgsql.cc +++ b/modules/gpgsqlbackend/spgsql.cc @@ -375,7 +375,7 @@ void SPgSQL::execute(const string& query) std::unique_ptr SPgSQL::prepare(const string& query, int nparams) { d_nstatements++; - return std::unique_ptr(new SPgSQLStatement(query, s_dolog, nparams, this, d_nstatements)); + return std::make_unique(query, s_dolog, nparams, this, d_nstatements); } void SPgSQL::startTransaction() diff --git a/modules/gsqlite3backend/gsqlite3backend.cc b/modules/gsqlite3backend/gsqlite3backend.cc index 6d9fae416dfa..f0df57213e40 100644 --- a/modules/gsqlite3backend/gsqlite3backend.cc +++ b/modules/gsqlite3backend/gsqlite3backend.cc @@ -121,7 +121,7 @@ class gSQLite3Factory : public BackendFactory declare(suffix, "update-account-query", "", "update domains set account=:account where name=:domain"); declare(suffix, "update-serial-query", "", "update domains set notified_serial=:serial where id=:domain_id"); declare(suffix, "update-lastcheck-query", "", "update domains set last_check=:last_check where id=:domain_id"); - declare(suffix, "info-all-master-query", "", "select domains.id, domains.name, domains.notified_serial, records.content from records join domains on records.name=domains.name where records.type='SOA' and records.disabled=0 and domains.type='MASTER'"); + declare(suffix, "info-all-master-query", "", "select domains.id, domains.name, domains.notified_serial, records.content from records join domains on records.domain_id=domains.id and records.name=domains.name where records.type='SOA' and records.disabled=0 and domains.type='MASTER'"); declare(suffix, "delete-domain-query", "", "delete from domains where name=:domain"); declare(suffix, "delete-zone-query", "", "delete from records where domain_id=:domain_id"); declare(suffix, "delete-rrset-query", "", "delete from records where domain_id=:domain_id and name=:qname and type=:qtype"); diff --git a/modules/lmdbbackend/lmdbbackend.cc b/modules/lmdbbackend/lmdbbackend.cc index 37458daf19c9..7c33213610f9 100644 --- a/modules/lmdbbackend/lmdbbackend.cc +++ b/modules/lmdbbackend/lmdbbackend.cc @@ -982,7 +982,7 @@ bool LMDBBackend::createDomain(const DNSName& domain, const DomainInfo::DomainKi return true; } -void LMDBBackend::getAllDomains(vector* domains, bool include_disabled) +void LMDBBackend::getAllDomains(vector* domains, bool doSerial, bool include_disabled) { domains->clear(); auto txn = d_tdomains->getROTransaction(); diff --git a/modules/lmdbbackend/lmdbbackend.hh b/modules/lmdbbackend/lmdbbackend.hh index a489770b6c5d..33399f31d70b 100644 --- a/modules/lmdbbackend/lmdbbackend.hh +++ b/modules/lmdbbackend/lmdbbackend.hh @@ -69,7 +69,7 @@ public: bool feedEnts3(int domain_id, const DNSName& domain, map& nonterm, const NSEC3PARAMRecordContent& ns3prc, bool narrow) override; bool replaceRRSet(uint32_t domain_id, const DNSName& qname, const QType& qt, const vector& rrset) override; - void getAllDomains(vector* domains, bool include_disabled = false) override; + void getAllDomains(vector* domains, bool doSerial, bool include_disabled) override; void lookup(const QType& type, const DNSName& qdomain, int zoneId, DNSPacket* p = nullptr) override; bool get(DNSResourceRecord& rr) override; bool get(DNSZoneRecord& dzr) override; diff --git a/modules/lua2backend/lua2api2.hh b/modules/lua2backend/lua2api2.hh index a17a90c5d9af..aaedf1a9d666 100644 --- a/modules/lua2backend/lua2api2.hh +++ b/modules/lua2backend/lua2api2.hh @@ -204,7 +204,7 @@ public: lookup_context_t ctx; if (p != NULL) { - ctx.emplace_back(lookup_context_t::value_type{"source_address", p->getRemote().toString()}); + ctx.emplace_back(lookup_context_t::value_type{"source_address", p->getInnerRemote().toString()}); ctx.emplace_back(lookup_context_t::value_type{"real_source_address", p->getRealRemote().toString()}); } @@ -298,7 +298,7 @@ public: return true; } - void getAllDomains(vector* domains, bool include_disabled = false) override + void getAllDomains(vector* domains, bool getSerial, bool include_disabled) override { if (f_get_all_domains == nullptr) return; diff --git a/modules/pipebackend/pipebackend.cc b/modules/pipebackend/pipebackend.cc index a5981c797d7e..5773adcf0bcc 100644 --- a/modules/pipebackend/pipebackend.cc +++ b/modules/pipebackend/pipebackend.cc @@ -66,10 +66,10 @@ void CoWrapper::launch() throw ArgException("pipe-command is not specified"); if (isUnixSocket(d_command)) { - d_cp = std::unique_ptr(new UnixRemote(d_command, d_timeout)); + d_cp = std::make_unique(d_command, d_timeout); } else { - auto coprocess = std::unique_ptr(new CoProcess(d_command, d_timeout)); + auto coprocess = std::make_unique(d_command, d_timeout); coprocess->launch(); d_cp = std::move(coprocess); } @@ -130,11 +130,11 @@ void PipeBackend::launch() try { if (!getArg("regex").empty()) { - d_regex = std::unique_ptr(new Regex(getArg("regex"))); + d_regex = std::make_unique(getArg("regex")); } d_regexstr = getArg("regex"); d_abiVersion = getArgAsNum("abi-version"); - d_coproc = unique_ptr(new CoWrapper(getArg("command"), getArgAsNum("timeout"), getArgAsNum("abi-version"))); + d_coproc = std::make_unique(getArg("command"), getArgAsNum("timeout"), getArgAsNum("abi-version")); } catch (const ArgException& A) { @@ -172,7 +172,7 @@ void PipeBackend::lookup(const QType& qtype, const DNSName& qname, int zoneId, D if (pkt_p) { localIP = pkt_p->getLocal().toString(); realRemote = pkt_p->getRealRemote(); - remoteIP = pkt_p->getRemote().toString(); + remoteIP = pkt_p->getInnerRemote().toString(); } // abi-version = 1 // type qname qclass qtype id remote-ip-address diff --git a/modules/randombackend/Makefile.am b/modules/randombackend/Makefile.am deleted file mode 100644 index 2da4bbde4a69..000000000000 --- a/modules/randombackend/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -pkglib_LTLIBRARIES = librandombackend.la - -EXTRA_DIST = OBJECTFILES OBJECTLIBS - -librandombackend_la_SOURCES = randombackend.cc -librandombackend_la_LDFLAGS = -module -avoid-version diff --git a/modules/randombackend/OBJECTFILES b/modules/randombackend/OBJECTFILES deleted file mode 100644 index 012f6073816b..000000000000 --- a/modules/randombackend/OBJECTFILES +++ /dev/null @@ -1 +0,0 @@ -randombackend.lo diff --git a/modules/randombackend/OBJECTLIBS b/modules/randombackend/OBJECTLIBS deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/modules/randombackend/randombackend.cc b/modules/randombackend/randombackend.cc deleted file mode 100644 index 01839e94fdbc..000000000000 --- a/modules/randombackend/randombackend.cc +++ /dev/null @@ -1,139 +0,0 @@ -/* - * This file is part of PowerDNS or dnsdist. - * Copyright -- PowerDNS.COM B.V. and its contributors - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * In addition, for the avoidance of any doubt, permission is granted to - * link this program with OpenSSL and to (re)distribute the binaries - * produced as the result of such linking. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif -#include "pdns/dnsbackend.hh" -#include "pdns/dns.hh" -#include "pdns/dnsbackend.hh" -#include "pdns/dnspacket.hh" -#include "pdns/dns_random.hh" -#include "pdns/pdnsexception.hh" -#include "pdns/logger.hh" -#include "pdns/version.hh" -#include - -/* FIRST PART */ -class RandomBackend : public DNSBackend -{ -public: - RandomBackend(const string& suffix = "") - { - setArgPrefix("random" + suffix); - d_ourname = DNSName(getArg("hostname")); - d_ourdomain = d_ourname; - d_ourdomain.chopOff(); - } - - bool list(const DNSName& target, int id, bool include_disabled) override - { - return false; // we don't support AXFR - } - - void lookup(const QType& type, const DNSName& qdomain, int zoneId, DNSPacket* p) override - { - if (qdomain == d_ourdomain) { - if (type.getCode() == QType::SOA || type.getCode() == QType::ANY) { - d_answer = "ns1." + d_ourdomain.toString() + " hostmaster." + d_ourdomain.toString() + " 1234567890 86400 7200 604800 300"; - } - else { - d_answer.clear(); - ; - } - } - else if (qdomain == d_ourname) { - if (type.getCode() == QType::A || type.getCode() == QType::ANY) { - ostringstream os; - os << dns_random(256) << "." << dns_random(256) << "." << dns_random(256) << "." << dns_random(256); - d_answer = os.str(); // our random ip address - } - else { - d_answer = ""; - } - } - else { - d_answer = ""; - } - } - - bool get(DNSResourceRecord& rr) override - { - if (d_answer.empty()) - return false; - - if (d_answer.find("ns1.") == 0) { - rr.qname = d_ourdomain; - rr.qtype = QType::SOA; - } - else { - rr.qname = d_ourname; - rr.qtype = QType::A; - } - rr.qclass = QClass::IN; // Internet class randomness. - rr.ttl = 5; // 5 seconds - rr.auth = true; // it may be random.. but it is auth! - rr.content = d_answer; - - d_answer.clear(); // this was the last answer - return true; - } - -private: - string d_answer; - DNSName d_ourname; - DNSName d_ourdomain; -}; - -/* SECOND PART */ - -class RandomFactory : public BackendFactory -{ -public: - RandomFactory() : - BackendFactory("random") {} - void declareArguments(const string& suffix = "") override - { - declare(suffix, "hostname", "Hostname which is to be random", "random.example.com"); - } - DNSBackend* make(const string& suffix = "") override - { - return new RandomBackend(suffix); - } -}; - -/* THIRD PART */ - -class RandomLoader -{ -public: - RandomLoader() - { - BackendMakers().report(new RandomFactory); - g_log << Logger::Info << "[randombackend] This is the random backend version " VERSION -#ifndef REPRODUCIBLE - << " (" __DATE__ " " __TIME__ ")" -#endif - << " reporting" << endl; - } -}; - -static RandomLoader randomLoader; diff --git a/modules/remotebackend/Makefile.am b/modules/remotebackend/Makefile.am index c544005905ea..1b5fc2002585 100644 --- a/modules/remotebackend/Makefile.am +++ b/modules/remotebackend/Makefile.am @@ -4,6 +4,14 @@ AM_CPPFLAGS += \ $(LIBCRYPTO_CFLAGS) \ $(LIBZMQ_CFLAGS) +if LUA +AM_CPPFLAGS +=$(LUA_CFLAGS) +endif + +if LIBSODIUM +AM_CPPFLAGS +=$(LIBSODIUM_CFLAGS) +endif + AM_LDFLAGS = $(THREADFLAGS) JSON11_LIBS = $(top_builddir)/ext/json11/libjson11.la @@ -110,6 +118,7 @@ libtestremotebackend_la_SOURCES = \ ../../pdns/dnsrecords.cc \ ../../pdns/dnssecinfra.cc \ ../../pdns/dnswriter.cc \ + ../../pdns/ednscookies.cc \ ../../pdns/ednsoptions.cc ../../pdns/ednsoptions.hh \ ../../pdns/ednssubnet.cc \ ../../pdns/iputils.cc \ @@ -146,6 +155,10 @@ libtestremotebackend_la_LDFLAGS = \ $(AM_LDFLAGS) \ $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) +if LIBSODIUM +libtestremotebackend_la_LIBADD += $(LIBSODIUM_LIBS) +endif + if REMOTEBACKEND_ZEROMQ libtestremotebackend_la_LIBADD += $(LIBZMQ_LIBS) endif diff --git a/modules/remotebackend/httpconnector.cc b/modules/remotebackend/httpconnector.cc index 99c0676e0edd..2c21d85c095f 100644 --- a/modules/remotebackend/httpconnector.cc +++ b/modules/remotebackend/httpconnector.cc @@ -385,7 +385,7 @@ int HTTPConnector::send_message(const Json& input) while (gAddrPtr) { try { - d_socket = std::unique_ptr(new Socket(gAddrPtr->ai_family, gAddrPtr->ai_socktype, gAddrPtr->ai_protocol)); + d_socket = std::make_unique(gAddrPtr->ai_family, gAddrPtr->ai_socktype, gAddrPtr->ai_protocol); d_addr.setSockaddr(gAddrPtr->ai_addr, gAddrPtr->ai_addrlen); d_socket->connect(d_addr); d_socket->setNonBlocking(); diff --git a/modules/remotebackend/remotebackend.cc b/modules/remotebackend/remotebackend.cc index fef3d0f6da21..380d956442de 100644 --- a/modules/remotebackend/remotebackend.cc +++ b/modules/remotebackend/remotebackend.cc @@ -165,20 +165,20 @@ int RemoteBackend::build() // connectors know what they are doing if (type == "unix") { - this->connector = std::unique_ptr(new UnixsocketConnector(options)); + this->connector = std::make_unique(options); } else if (type == "http") { - this->connector = std::unique_ptr(new HTTPConnector(options)); + this->connector = std::make_unique(options); } else if (type == "zeromq") { #ifdef REMOTEBACKEND_ZEROMQ - this->connector = std::unique_ptr(new ZeroMQConnector(options)); + this->connector = std::make_unique(options); #else throw PDNSException("Invalid connection string: zeromq connector support not enabled. Recompile with --enable-remotebackend-zeromq"); #endif } else if (type == "pipe") { - this->connector = std::unique_ptr(new PipeConnector(options)); + this->connector = std::make_unique(options); } else { throw PDNSException("Invalid connection string: unknown connector"); @@ -203,7 +203,7 @@ void RemoteBackend::lookup(const QType& qtype, const DNSName& qdomain, int zoneI if (pkt_p) { localIP = pkt_p->getLocal().toString(); realRemote = pkt_p->getRealRemote().toString(); - remoteIP = pkt_p->getRemote().toString(); + remoteIP = pkt_p->getInnerRemote().toString(); } Json query = Json::object{ @@ -858,7 +858,7 @@ bool RemoteBackend::searchComments(const string& pattern, int maxResults, vector return false; } -void RemoteBackend::getAllDomains(vector* domains, bool include_disabled) +void RemoteBackend::getAllDomains(vector* domains, bool getSerial, bool include_disabled) { Json query = Json::object{ {"method", "getAllDomains"}, diff --git a/modules/remotebackend/remotebackend.hh b/modules/remotebackend/remotebackend.hh index 7832682a1c6c..08811fe078ea 100644 --- a/modules/remotebackend/remotebackend.hh +++ b/modules/remotebackend/remotebackend.hh @@ -198,7 +198,7 @@ public: string directBackendCmd(const string& querystr) override; bool searchRecords(const string& pattern, int maxResults, vector& result) override; bool searchComments(const string& pattern, int maxResults, vector& result) override; - void getAllDomains(vector* domains, bool include_disabled = false) override; + void getAllDomains(vector* domains, bool getSerial, bool include_disabled) override; void getUpdatedMasters(vector* domains) override; void alsoNotifies(const DNSName& domain, set* ips) override; void getUnfreshSlaveInfos(vector* domains) override; diff --git a/modules/remotebackend/test-remotebackend.cc b/modules/remotebackend/test-remotebackend.cc index 084ce1ada3a8..06ebeb596eb2 100644 --- a/modules/remotebackend/test-remotebackend.cc +++ b/modules/remotebackend/test-remotebackend.cc @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(test_method_getAllDomains) BOOST_TEST_MESSAGE("Testing getAllDomains method"); vector result; - be->getAllDomains(&result, true); + be->getAllDomains(&result, true, true); di = result[0]; BOOST_CHECK_EQUAL(di.zone.toString(), "unit.test."); diff --git a/modules/tinydnsbackend/tinydnsbackend.cc b/modules/tinydnsbackend/tinydnsbackend.cc index 0dbcf85d1654..ae35c0feaaad 100644 --- a/modules/tinydnsbackend/tinydnsbackend.cc +++ b/modules/tinydnsbackend/tinydnsbackend.cc @@ -29,8 +29,7 @@ static string backendname = "[TinyDNSBackend] "; uint32_t TinyDNSBackend::s_lastId; -std::mutex TinyDNSBackend::s_domainInfoLock; -TinyDNSBackend::TDI_suffix_t TinyDNSBackend::s_domainInfo; +LockGuarded TinyDNSBackend::s_domainInfo; vector TinyDNSBackend::getLocations() { @@ -59,7 +58,7 @@ vector TinyDNSBackend::getLocations() for (int i = 4; i >= 0; i--) { string searchkey(key, i + 2); try { - auto reader = std::unique_ptr(new CDB(getArg("dbfile"))); + auto reader = std::make_unique(getArg("dbfile")); ret = reader->findall(searchkey); } catch (const std::exception& e) { @@ -91,17 +90,16 @@ TinyDNSBackend::TinyDNSBackend(const string& suffix) void TinyDNSBackend::getUpdatedMasters(vector* retDomains) { - std::lock_guard l(s_domainInfoLock); //TODO: We could actually lock less if we do it per suffix. - - if (!s_domainInfo.count(d_suffix)) { + auto domainInfo = s_domainInfo.lock(); //TODO: We could actually lock less if we do it per suffix. + if (!domainInfo->count(d_suffix)) { TDI_t tmp; - s_domainInfo.insert(make_pair(d_suffix, tmp)); + domainInfo->emplace(d_suffix, tmp); } - TDI_t* domains = &s_domainInfo[d_suffix]; + TDI_t* domains = &(*domainInfo)[d_suffix]; vector allDomains; - getAllDomains(&allDomains); + getAllDomains(&allDomains, true, false); if (domains->size() == 0 && !mustDo("notify-on-startup")) { for (vector::iterator di = allDomains.begin(); di != allDomains.end(); ++di) { di->notified_serial = 0; @@ -136,11 +134,11 @@ void TinyDNSBackend::getUpdatedMasters(vector* retDomains) void TinyDNSBackend::setNotified(uint32_t id, uint32_t serial) { - std::lock_guard l(s_domainInfoLock); - if (!s_domainInfo.count(d_suffix)) { + auto domainInfo = s_domainInfo.lock(); + if (!domainInfo->count(d_suffix)) { throw PDNSException("Can't get list of domains to set the serial."); } - TDI_t* domains = &s_domainInfo[d_suffix]; + TDI_t* domains = &(*domainInfo)[d_suffix]; TDIById_t& domain_index = domains->get(); TDIById_t::iterator itById = domain_index.find(id); if (itById == domain_index.end()) { @@ -150,17 +148,17 @@ void TinyDNSBackend::setNotified(uint32_t id, uint32_t serial) DLOG(g_log << Logger::Debug << backendname << "Setting serial for " << itById->zone << " to " << serial << endl); domain_index.modify(itById, TDI_SerialModifier(serial)); } - s_domainInfo[d_suffix] = *domains; + (*domainInfo)[d_suffix] = *domains; } -void TinyDNSBackend::getAllDomains(vector* domains, bool include_disabled) +void TinyDNSBackend::getAllDomains(vector* domains, bool getSerial, bool include_disabled) { d_isAxfr = true; d_isGetDomains = true; d_dnspacket = NULL; try { - d_cdbReader = std::unique_ptr(new CDB(getArg("dbfile"))); + d_cdbReader = std::make_unique(getArg("dbfile")); } catch (const std::exception& e) { g_log << Logger::Error << e.what() << endl; @@ -173,17 +171,25 @@ void TinyDNSBackend::getAllDomains(vector* domains, bool include_dis while (get(rr)) { if (rr.qtype.getCode() == QType::SOA && dupcheck.insert(rr.qname).second) { - SOAData sd; - fillSOAData(rr.content, sd); - DomainInfo di; di.id = -1; //TODO: Check if this is ok. di.backend = this; di.zone = rr.qname; - di.serial = sd.serial; - di.notified_serial = sd.serial; di.kind = DomainInfo::Master; di.last_check = time(0); + + if (getSerial) { + SOAData sd; + try { + fillSOAData(rr.content, sd); + di.serial = sd.serial; + } + catch (...) { + di.serial = 0; + } + } + + di.notified_serial = di.serial; domains->push_back(di); } } @@ -195,7 +201,7 @@ bool TinyDNSBackend::list(const DNSName& target, int domain_id, bool include_dis d_isGetDomains = false; string key = target.toDNSStringLC(); try { - d_cdbReader = std::unique_ptr(new CDB(getArg("dbfile"))); + d_cdbReader = std::make_unique(getArg("dbfile")); } catch (const std::exception& e) { g_log << Logger::Error << e.what() << endl; @@ -225,7 +231,7 @@ void TinyDNSBackend::lookup(const QType& qtype, const DNSName& qdomain, int zone d_qtype = qtype; try { - d_cdbReader = std::unique_ptr(new CDB(getArg("dbfile"))); + d_cdbReader = std::make_unique(getArg("dbfile")); } catch (const std::exception& e) { g_log << Logger::Error << e.what() << endl; diff --git a/modules/tinydnsbackend/tinydnsbackend.hh b/modules/tinydnsbackend/tinydnsbackend.hh index efcb2436a4e8..10ee5527825c 100644 --- a/modules/tinydnsbackend/tinydnsbackend.hh +++ b/modules/tinydnsbackend/tinydnsbackend.hh @@ -32,7 +32,6 @@ #include #include #include -#include using namespace ::boost; using namespace ::boost::multi_index; @@ -71,7 +70,7 @@ public: void lookup(const QType& qtype, const DNSName& qdomain, int zoneId, DNSPacket* pkt_p = nullptr) override; bool list(const DNSName& target, int domain_id, bool include_disabled = false) override; bool get(DNSResourceRecord& rr) override; - void getAllDomains(vector* domains, bool include_disabled = false) override; + void getAllDomains(vector* domains, bool getSerial, bool include_disabled) override; //Master mode operation void getUpdatedMasters(vector* domains) override; @@ -110,7 +109,6 @@ private: string d_suffix; // Statics - static std::mutex s_domainInfoLock; - static TDI_suffix_t s_domainInfo; + static LockGuarded s_domainInfo; static uint32_t s_lastId; // used to give a domain an id. }; diff --git a/pdns/Makefile.am b/pdns/Makefile.am index a17aa95e0aea..004c8becfedc 100644 --- a/pdns/Makefile.am +++ b/pdns/Makefile.am @@ -17,6 +17,7 @@ AM_CXXFLAGS = \ AM_LDFLAGS = \ $(PROGRAM_LDFLAGS) \ + $(LIBCRYPTO_LIBS) \ $(THREADFLAGS) AM_LFLAGS = -i @@ -200,6 +201,7 @@ pdns_server_SOURCES = \ comment.hh \ common_startup.cc common_startup.hh \ communicator.cc communicator.hh \ + credentials.cc credentials.hh \ dbdnsseckeeper.cc \ digests.hh \ distributor.hh \ @@ -219,6 +221,7 @@ pdns_server_SOURCES = \ dynhandler.cc dynhandler.hh \ dynlistener.cc dynlistener.hh \ dynmessenger.hh \ + ednscookies.cc ednscookies.hh \ ednsoptions.cc ednsoptions.hh \ ednssubnet.cc ednssubnet.hh \ histogram.hh \ @@ -239,6 +242,7 @@ pdns_server_SOURCES = \ packetcache.hh \ packethandler.cc packethandler.hh \ pdnsexception.hh \ + proxy-protocol.cc proxy-protocol.hh \ qtype.cc qtype.hh \ query-local-address.hh query-local-address.cc \ rcpgenerator.cc \ @@ -254,6 +258,7 @@ pdns_server_SOURCES = \ signingpipe.cc signingpipe.hh \ sillyrecords.cc \ slavecommunicator.cc \ + stat_t.hh \ statbag.cc statbag.hh \ stubresolver.cc stubresolver.hh \ svc-records.cc svc-records.hh \ @@ -332,6 +337,7 @@ pdnsutil_SOURCES = \ bindparser.yy \ cachecleaner.hh \ circular_buffer.hh \ + credentials.cc credentials.hh \ dbdnsseckeeper.cc \ dns.cc \ dns_random.cc \ @@ -345,6 +351,7 @@ pdnsutil_SOURCES = \ dnssecsigner.cc \ dnswriter.cc dnswriter.hh \ dynlistener.cc \ + ednscookies.cc ednscookies.hh \ ednsoptions.cc ednsoptions.hh \ ednssubnet.cc \ ipcipher.cc ipcipher.hh \ @@ -675,6 +682,7 @@ ixfrdist_SOURCES = \ axfr-retriever.cc \ base32.cc \ base64.cc base64.hh \ + credentials.cc credentials.hh \ dns.cc \ dns_random_urandom.cc dns_random.hh \ dnslabeltext.cc \ @@ -709,7 +717,6 @@ ixfrdist_SOURCES = \ webserver.hh webserver.cc \ zoneparser-tng.cc - ixfrdist_LDADD = \ $(BOOST_PROGRAM_OPTIONS_LIBS) \ $(JSON11_LIBS) \ @@ -722,6 +729,10 @@ ixfrdist_LDFLAGS = \ $(BOOST_PROGRAM_OPTIONS_LDFLAGS) \ $(LIBCRYPTO_LDFLAGS) +if LIBSODIUM +ixfrdist_LDADD += $(LIBSODIUM_LIBS) +endif + if PKCS11 ixfrdist_SOURCES += pkcs11signers.cc pkcs11signers.hh ixfrdist_LDADD += $(P11KIT1_LIBS) @@ -908,8 +919,11 @@ tsig_tests_LDADD += $(P11KIT1_LIBS) endif speedtest_SOURCES = \ + arguments.cc arguments.hh \ base32.cc \ base64.cc base64.hh \ + credentials.cc credentials.hh \ + dns_random.cc dns_random.hh \ dnslabeltext.cc \ dnsname.cc dnsname.hh \ dnsparser.cc dnsparser.hh \ @@ -1312,6 +1326,7 @@ testrunner_SOURCES = \ base64.cc \ bindlexer.l \ bindparser.yy \ + credentials.cc credentials.hh \ dbdnsseckeeper.cc \ dns.cc \ dns_random.cc \ @@ -1345,9 +1360,10 @@ testrunner_SOURCES = \ qtype.cc \ rcpgenerator.cc \ responsestats-auth.cc \ - responsestats.cc \ + responsestats.cc responsestats.hh \ shuffle.cc shuffle.hh \ sillyrecords.cc \ + stat_t.hh \ statbag.cc \ stubresolver.hh stubresolver.cc \ svc-records.cc svc-records.hh \ @@ -1358,6 +1374,7 @@ testrunner_SOURCES = \ test-bindparser_cc.cc \ test-common.hh \ test-communicator_hh.cc \ + test-credentials_cc.cc \ test-digests_hh.cc \ test-distributor_hh.cc \ test-dns_random_hh.cc \ @@ -1367,6 +1384,7 @@ testrunner_SOURCES = \ test-dnsrecordcontent.cc \ test-dnsrecords_cc.cc \ test-dnswriter_cc.cc \ + test-ednscookie_cc.cc \ test-ipcrypt_cc.cc \ test-iputils_hh.cc \ test-ixfr_cc.cc \ @@ -1419,6 +1437,7 @@ endif if LIBSODIUM testrunner_SOURCES += sodiumsigners.cc testrunner_LDADD += $(LIBSODIUM_LIBS) +speedtest_LDADD += $(LIBSODIUM_LIBS) endif if LIBDECAF @@ -1462,6 +1481,10 @@ pdns_control_SOURCES = \ statbag.cc \ unix_utility.cc +pdns_control_LDFLAGS = \ + $(AM_LDFLAGS) \ + $(LIBCRYPTO_LDFLAGS) + if UNIT_TESTS noinst_PROGRAMS = testrunner if HAVE_BOOST_GE_148 @@ -1563,7 +1586,7 @@ fuzz_target_dnsdistcache_SOURCES = \ dnsdist-cache.cc dnsdist-cache.hh \ dnsdist-ecs.cc dnsdist-ecs.hh \ dnsdist-idstate.hh \ - dnsdist-protocols.hh \ + dnsdist-protocols.cc dnsdist-protocols.hh \ dnslabeltext.cc \ dnsname.cc dnsname.hh \ dnsparser.cc dnsparser.hh \ @@ -1641,8 +1664,10 @@ if HAVE_SYSTEMD pdns.service: pdns.service.in $(AM_V_GEN)sed -e 's![@]sbindir[@]!$(sbindir)!' -e 's![@]service_user[@]!$(service_user)!' -e 's![@]service_group[@]!$(service_group)!' < $< > $@ if !HAVE_SYSTEMD_WITH_RUNTIME_DIR_ENV +if HAVE_SYSTEMD_PERCENT_T $(AM_V_GEN)sed -e 's!/pdns_server!& --socket-dir=%t/pdns!' -i $@ endif +endif if !HAVE_SYSTEMD_LOCK_PERSONALITY $(AM_V_GEN)perl -ni -e 'print unless /^LockPersonality/' $@ endif @@ -1701,12 +1726,14 @@ endif pdns@.service: pdns.service $(AM_V_GEN)sed -e 's!/pdns_server!& --config-name=%i!' \ -e 's!Authoritative Server!& %i!' \ - -e 's!RuntimeDirectory=.*!&-%i!' \ -e 's!SyslogIdentifier=.*!&-%i!' \ < $< > $@ +if HAVE_SYSTEMD_PERCENT_T + $(AM_V_GEN)sed -e 's!RuntimeDirectory=.*!&-%i!' -i $@ if !HAVE_SYSTEMD_WITH_RUNTIME_DIR_ENV $(AM_V_GEN)sed -e 's!--socket-dir=[^ ]\+!&-%i !' -i $@ endif +endif systemdsystemunitdir = $(SYSTEMD_DIR) diff --git a/pdns/auth-packetcache.cc b/pdns/auth-packetcache.cc index e31a0182eff7..0a4a835c09f1 100644 --- a/pdns/auth-packetcache.cc +++ b/pdns/auth-packetcache.cc @@ -43,24 +43,10 @@ AuthPacketCache::AuthPacketCache(size_t mapsCount): d_maps(mapsCount), d_lastcle d_statnumentries=S.getPointer("packetcache-size"); } -AuthPacketCache::~AuthPacketCache() -{ - try { - vector locks; - for(auto& mc : d_maps) { - locks.push_back(WriteLock(mc.d_mut)); - } - locks.clear(); - } - catch(...) { - } -} - void AuthPacketCache::MapCombo::reserve(size_t numberOfEntries) { #if BOOST_VERSION >= 105600 - WriteLock wl(&d_mut); - d_map.get().reserve(numberOfEntries); + d_map.write_lock()->get().reserve(numberOfEntries); #endif /* BOOST_VERSION >= 105600 */ } @@ -72,7 +58,8 @@ bool AuthPacketCache::get(DNSPacket& p, DNSPacket& cached) cleanupIfNeeded(); - uint32_t hash = canHashPacket(p.getString(), /* don't skip ECS */ false); + static const std::unordered_set optionsToSkip{ EDNSOptionCode::COOKIE}; + uint32_t hash = canHashPacket(p.getString(), /* don't skip ECS */optionsToSkip); p.setHash(hash); string value; @@ -80,13 +67,13 @@ bool AuthPacketCache::get(DNSPacket& p, DNSPacket& cached) time_t now = time(nullptr); auto& mc = getMap(p.qdomain); { - TryReadLock rl(&mc.d_mut); - if(!rl.gotIt()) { + auto map = mc.d_map.try_read_lock(); + if (!map.owns_lock()) { S.inc("deferred-packetcache-lookup"); return false; } - haveSomething = getEntryLocked(mc.d_map, p.getString(), hash, p.qdomain, p.qtype.getCode(), p.d_tcp, now, value); + haveSomething = getEntryLocked(*map, p.getString(), hash, p.qdomain, p.qtype.getCode(), p.d_tcp, now, value); } if (!haveSomething) { @@ -146,13 +133,13 @@ void AuthPacketCache::insert(DNSPacket& q, DNSPacket& r, unsigned int maxTTL) auto& mc = getMap(entry.qname); { - TryWriteLock l(&mc.d_mut); - if (!l.gotIt()) { + auto map = mc.d_map.try_write_lock(); + if (!map.owns_lock()) { S.inc("deferred-packetcache-inserts"); return; } - auto& idx = mc.d_map.get(); + auto& idx = map->get(); auto range = idx.equal_range(hash); auto iter = range.first; @@ -161,7 +148,7 @@ void AuthPacketCache::insert(DNSPacket& q, DNSPacket& r, unsigned int maxTTL) continue; } - moveCacheItemToBack(mc.d_map, iter); + moveCacheItemToBack(*map, iter); iter->value = entry.value; iter->ttd = now + ourttl; iter->created = now; @@ -169,11 +156,11 @@ void AuthPacketCache::insert(DNSPacket& q, DNSPacket& r, unsigned int maxTTL) } /* no existing entry found to refresh */ - mc.d_map.insert(std::move(entry)); + map->insert(std::move(entry)); if (*d_statnumentries >= d_maxEntries) { /* remove the least recently inserted or replaced entry */ - auto& sidx = mc.d_map.get(); + auto& sidx = map->get(); sidx.pop_front(); } else { @@ -182,7 +169,7 @@ void AuthPacketCache::insert(DNSPacket& q, DNSPacket& r, unsigned int maxTTL) } } -bool AuthPacketCache::getEntryLocked(cmap_t& map, const std::string& query, uint32_t hash, const DNSName &qname, uint16_t qtype, bool tcp, time_t now, string& value) +bool AuthPacketCache::getEntryLocked(const cmap_t& map, const std::string& query, uint32_t hash, const DNSName &qname, uint16_t qtype, bool tcp, time_t now, string& value) { auto& idx = map.get(); auto range = idx.equal_range(hash); diff --git a/pdns/auth-packetcache.hh b/pdns/auth-packetcache.hh index bceab073de59..618041cf17d0 100644 --- a/pdns/auth-packetcache.hh +++ b/pdns/auth-packetcache.hh @@ -48,7 +48,6 @@ class AuthPacketCache : public PacketCache { public: AuthPacketCache(size_t mapsCount=1024); - ~AuthPacketCache(); void insert(DNSPacket& q, DNSPacket& r, uint32_t maxTTL); //!< We copy the contents of *p into our cache. Do not needlessly call this to insert questions already in the cache as it wastes resources @@ -116,8 +115,7 @@ private: void reserve(size_t numberOfEntries); - ReadWriteLock d_mut; - cmap_t d_map; + SharedLockGuarded d_map; }; vector d_maps; @@ -127,7 +125,7 @@ private: } static bool entryMatches(cmap_t::index::type::iterator& iter, const std::string& query, const DNSName& qname, uint16_t qtype, bool tcp); - bool getEntryLocked(cmap_t& map, const std::string& query, uint32_t hash, const DNSName &qname, uint16_t qtype, bool tcp, time_t now, string& entry); + bool getEntryLocked(const cmap_t& map, const std::string& query, uint32_t hash, const DNSName &qname, uint16_t qtype, bool tcp, time_t now, string& entry); void cleanupIfNeeded(); AtomicCounter d_ops{0}; diff --git a/pdns/auth-querycache.cc b/pdns/auth-querycache.cc index 7812a82e1ff6..6913296d847f 100644 --- a/pdns/auth-querycache.cc +++ b/pdns/auth-querycache.cc @@ -44,24 +44,10 @@ AuthQueryCache::AuthQueryCache(size_t mapsCount): d_maps(mapsCount), d_lastclean d_statnumentries=S.getPointer("query-cache-size"); } -AuthQueryCache::~AuthQueryCache() -{ - try { - vector locks; - for(auto& mc : d_maps) { - locks.push_back(WriteLock(mc.d_mut)); - } - locks.clear(); - } - catch(...) { - } -} - void AuthQueryCache::MapCombo::reserve(size_t numberOfEntries) { #if BOOST_VERSION >= 105600 - WriteLock wl(&d_mut); - d_map.get().reserve(numberOfEntries); + d_map.write_lock()->get().reserve(numberOfEntries); #endif /* BOOST_VERSION >= 105600 */ } @@ -74,13 +60,13 @@ bool AuthQueryCache::getEntry(const DNSName &qname, const QType& qtype, vectorinsert(val); if (!inserted) { - mc.d_map.replace(place, std::move(val)); - moveCacheItemToBack(mc.d_map, place); + map->replace(place, std::move(val)); + moveCacheItemToBack(*map, place); } else { if (*d_statnumentries >= d_maxEntries) { /* remove the least recently inserted or replaced entry */ - auto& sidx = mc.d_map.get(); + auto& sidx = map->get(); sidx.pop_front(); } else { @@ -130,7 +116,7 @@ void AuthQueryCache::insert(const DNSName &qname, const QType& qtype, vector& value, int zoneID, time_t now) +bool AuthQueryCache::getEntryLocked(const cmap_t& map, const DNSName &qname, uint16_t qtype, vector& value, int zoneID, time_t now) { auto& idx = boost::multi_index::get(map); auto iter = idx.find(tie(qname, qtype, zoneID)); @@ -155,9 +141,9 @@ map AuthQueryCache::getCounts() uint64_t queryCacheEntries=0, negQueryCacheEntries=0; for(auto& mc : d_maps) { - ReadLock l(&mc.d_mut); + auto map = mc.d_map.read_lock(); - for(const auto & iter : mc.d_map) { + for(const auto & iter : *map) { if(iter.drs.empty()) negQueryCacheEntries++; else diff --git a/pdns/auth-querycache.hh b/pdns/auth-querycache.hh index 33bd8a7cae0a..b5ee72db123d 100644 --- a/pdns/auth-querycache.hh +++ b/pdns/auth-querycache.hh @@ -37,7 +37,6 @@ class AuthQueryCache : public boost::noncopyable { public: AuthQueryCache(size_t mapsCount=1024); - ~AuthQueryCache(); void insert(const DNSName &qname, const QType& qtype, vector&& content, uint32_t ttl, int zoneID); @@ -99,8 +98,7 @@ private: void reserve(size_t numberOfEntries); - ReadWriteLock d_mut; - cmap_t d_map; + SharedLockGuarded d_map; }; vector d_maps; @@ -109,7 +107,7 @@ private: return d_maps[qname.hash() % d_maps.size()]; } - bool getEntryLocked(cmap_t& map, const DNSName &content, uint16_t qtype, vector& entry, int zoneID, time_t now); + bool getEntryLocked(const cmap_t& map, const DNSName &content, uint16_t qtype, vector& entry, int zoneID, time_t now); void cleanupIfNeeded(); AtomicCounter d_ops{0}; diff --git a/pdns/auth-zonecache.cc b/pdns/auth-zonecache.cc index bdac5ed81ab5..f5e5adec3b18 100644 --- a/pdns/auth-zonecache.cc +++ b/pdns/auth-zonecache.cc @@ -42,27 +42,14 @@ AuthZoneCache::AuthZoneCache(size_t mapsCount) : d_statnumentries = S.getPointer("zone-cache-size"); } -AuthZoneCache::~AuthZoneCache() -{ - try { - vector locks; - for (auto& mc : d_maps) { - locks.push_back(WriteLock(mc.d_mut)); - } - locks.clear(); - } - catch (...) { - } -} - bool AuthZoneCache::getEntry(const DNSName& zone, int& zoneId) { auto& mc = getMap(zone); bool found = false; { - ReadLock rl(mc.d_mut); - auto iter = mc.d_map.find(zone); - if (iter != mc.d_map.end()) { + auto map = mc.d_map.read_lock(); + auto iter = map->find(zone); + if (iter != map->end()) { found = true; zoneId = iter->second.zoneId; } @@ -93,7 +80,7 @@ void AuthZoneCache::replace(const vector>& zone_indices) return; size_t count = zone_indices.size(); - vector newMaps(d_maps.size()); + vector newMaps(d_maps.size()); // build new maps for (const tuple& tup : zone_indices) { @@ -101,36 +88,36 @@ void AuthZoneCache::replace(const vector>& zone_indices) CacheValue val; val.zoneId = tup.get<1>(); auto& mc = newMaps[getMapIndex(zone)]; - auto iter = mc.d_map.find(zone); - if (iter != mc.d_map.end()) { + auto iter = mc.find(zone); + if (iter != mc.end()) { iter->second = std::move(val); } else { - mc.d_map.emplace(zone, val); + mc.emplace(zone, val); } } { - WriteLock globalLock(d_mut); - if (d_replacePending) { + auto pending = d_pending.lock(); + if (pending->d_replacePending) { // add/replace all zones created while data collection for replace() was already in progress. - for (const tuple& tup : d_pendingAdds) { + for (const tuple& tup : pending->d_pendingAdds) { const DNSName& zone = tup.get<0>(); CacheValue val; val.zoneId = tup.get<1>(); auto& mc = newMaps[getMapIndex(zone)]; - mc.d_map[zone] = val; + mc[zone] = val; } } for (size_t mapIndex = 0; mapIndex < d_maps.size(); mapIndex++) { auto& mc = d_maps[mapIndex]; - WriteLock mcLock(mc.d_mut); - mc.d_map = std::move(newMaps[mapIndex].d_map); + auto map = mc.d_map.write_lock(); + *map = std::move(newMaps[mapIndex]); } - d_pendingAdds.clear(); - d_replacePending = false; + pending->d_pendingAdds.clear(); + pending->d_replacePending = false; } d_statnumentries->store(count); @@ -142,9 +129,9 @@ void AuthZoneCache::add(const DNSName& zone, const int zoneId) return; { - WriteLock globalLock(d_mut); - if (d_replacePending) { - d_pendingAdds.push_back({zone, zoneId}); + auto pending = d_pending.lock(); + if (pending->d_replacePending) { + pending->d_pendingAdds.emplace_back(zone, zoneId); } } @@ -154,13 +141,13 @@ void AuthZoneCache::add(const DNSName& zone, const int zoneId) int mapIndex = getMapIndex(zone); { auto& mc = d_maps[mapIndex]; - WriteLock mcLock(mc.d_mut); - auto iter = mc.d_map.find(zone); - if (iter != mc.d_map.end()) { + auto map = mc.d_map.write_lock(); + auto iter = map->find(zone); + if (iter != map->end()) { iter->second = std::move(val); } else { - mc.d_map.emplace(zone, val); + map->emplace(zone, val); } } } @@ -171,8 +158,8 @@ void AuthZoneCache::setReplacePending() return; { - WriteLock globalLock(d_mut); - d_replacePending = true; - d_pendingAdds.clear(); + auto pending = d_pending.lock(); + pending->d_replacePending = true; + pending->d_pendingAdds.clear(); } } diff --git a/pdns/auth-zonecache.hh b/pdns/auth-zonecache.hh index e5ace8b265e1..e3968e1f8937 100644 --- a/pdns/auth-zonecache.hh +++ b/pdns/auth-zonecache.hh @@ -31,7 +31,6 @@ class AuthZoneCache : public boost::noncopyable { public: AuthZoneCache(size_t mapsCount = 1024); - ~AuthZoneCache(); void replace(const vector>& zone); void add(const DNSName& zone, const int zoneId); @@ -70,8 +69,7 @@ private: MapCombo(const MapCombo&) = delete; MapCombo& operator=(const MapCombo&) = delete; - ReadWriteLock d_mut; - cmap_t d_map; + SharedLockGuarded d_map; }; vector d_maps; @@ -90,9 +88,12 @@ private: time_t d_refreshinterval{0}; - ReadWriteLock d_mut; - std::vector> d_pendingAdds; - bool d_replacePending{false}; + struct PendingData + { + std::vector> d_pendingAdds; + bool d_replacePending{false}; + }; + LockGuarded d_pending; }; extern AuthZoneCache g_zoneCache; diff --git a/pdns/backends/gsql/gsqlbackend.cc b/pdns/backends/gsql/gsqlbackend.cc index 51cece2f4717..be0c594155f4 100644 --- a/pdns/backends/gsql/gsqlbackend.cc +++ b/pdns/backends/gsql/gsqlbackend.cc @@ -1397,7 +1397,7 @@ bool GSQLBackend::deleteDomain(const DNSName &domain) return true; } -void GSQLBackend::getAllDomains(vector *domains, bool include_disabled) +void GSQLBackend::getAllDomains(vector* domains, bool getSerial, bool include_disabled) { DLOG(g_log<<"GSQLBackend retrieving all domains."< *domains, bool include_disabl } } - if(!row[2].empty()) { + if (getSerial && !row[2].empty()) { SOAData sd; - fillSOAData(row[2], sd); - di.serial = sd.serial; + try { + fillSOAData(row[2], sd); + di.serial = sd.serial; + } + catch (...) { + di.serial = 0; + } } + try { di.notified_serial = pdns_stou(row[5]); di.last_check = pdns_stou(row[6]); } catch(...) { continue; } + di.account = row[7]; di.backend = this; - + domains->push_back(di); } d_getAllDomainsQuery_stmt->reset(); diff --git a/pdns/backends/gsql/gsqlbackend.hh b/pdns/backends/gsql/gsqlbackend.hh index 49dde6050888..66fc76757909 100644 --- a/pdns/backends/gsql/gsqlbackend.hh +++ b/pdns/backends/gsql/gsqlbackend.hh @@ -185,7 +185,7 @@ public: void lookup(const QType &, const DNSName &qdomain, int zoneId, DNSPacket *p=nullptr) override; bool list(const DNSName &target, int domain_id, bool include_disabled=false) override; bool get(DNSResourceRecord &r) override; - void getAllDomains(vector *domains, bool include_disabled=false) override; + void getAllDomains(vector* domains, bool getSerial, bool include_disabled) override; void alsoNotifies(const DNSName &domain, set *ips) override; bool startTransaction(const DNSName &domain, int domain_id=-1) override; bool commitTransaction() override; diff --git a/pdns/bindparserclasses.hh b/pdns/bindparserclasses.hh index 177a004da758..b47d23827403 100644 --- a/pdns/bindparserclasses.hh +++ b/pdns/bindparserclasses.hh @@ -57,7 +57,7 @@ public: bool operator<(const BindDomainInfo& b) const { - return make_pair(d_dev, d_ino) < make_pair(b.d_dev, b.d_ino); + return pair(d_dev, d_ino) < pair(b.d_dev, b.d_ino); } }; diff --git a/pdns/bpf-filter.cc b/pdns/bpf-filter.cc index 59912dbeb16b..29bd6fe31349 100644 --- a/pdns/bpf-filter.cc +++ b/pdns/bpf-filter.cc @@ -47,6 +47,50 @@ int bpf_create_map(enum bpf_map_type map_type, int key_size, int value_size, return syscall(SYS_bpf, BPF_MAP_CREATE, &attr, sizeof(attr)); } +int bpf_pin_map(int fd, const std::string& path) +{ + union bpf_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.bpf_fd = fd; + attr.pathname = ptr_to_u64(const_cast(path.c_str())); + return syscall(SYS_bpf, BPF_OBJ_PIN, &attr, sizeof(attr)); +} + +int bpf_load_pinned_map(const std::string& path) +{ + union bpf_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.pathname = ptr_to_u64(const_cast(path.c_str())); + return syscall(SYS_bpf, BPF_OBJ_GET, &attr, sizeof(attr)); +} + +void bpf_check_map_sizes(int fd, uint32_t expectedKeySize, uint32_t expectedValueSize) +{ + struct bpf_map_info info; + uint32_t info_len = sizeof(info); + memset(&info, 0, sizeof(info)); + + union bpf_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.info.bpf_fd = fd; + attr.info.info_len = info_len; + attr.info.info = ptr_to_u64(&info); + + int err = syscall(SYS_bpf, BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)); + if (err != 0) { + throw std::runtime_error("Error checking the size of eBPF map: " + stringerror()); + } + if (info_len != sizeof(info)) { + throw std::runtime_error("Error checking the size of eBPF map: invalid info size returned"); + } + if (info.key_size != expectedKeySize) { + throw std::runtime_error("Error checking the size of eBPF map: key size mismatch (" + std::to_string(info.key_size) + " VS " + std::to_string(expectedKeySize) + ")"); + } + if (info.value_size != expectedValueSize) { + throw std::runtime_error("Error checking the size of eBPF map: value size mismatch (" + std::to_string(info.value_size) + " VS " + std::to_string(expectedValueSize) + ")"); + } +} + int bpf_update_elem(int fd, void *key, void *value, unsigned long long flags) { union bpf_attr attr; @@ -88,8 +132,8 @@ int bpf_get_next_key(int fd, void *key, void *next_key) } int bpf_prog_load(enum bpf_prog_type prog_type, - const struct bpf_insn *insns, int prog_len, - const char *license, int kern_version) + const struct bpf_insn *insns, int prog_len, + const char *license, int kern_version) { char log_buf[65535]; union bpf_attr attr; @@ -133,71 +177,195 @@ struct QNameKey uint8_t qname[255]; }; -struct QNameValue +struct QNameAndQTypeKey { - uint64_t counter; + uint8_t qname[255]; uint16_t qtype; }; -BPFFilter::BPFFilter(uint32_t maxV4Addresses, uint32_t maxV6Addresses, uint32_t maxQNames): d_maps(Maps()), d_maxV4(maxV4Addresses), d_maxV6(maxV6Addresses), d_maxQNames(maxQNames) +struct QNameValue { - auto maps = d_maps.lock(); - maps->d_v4map.fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(uint32_t), sizeof(uint64_t), (int) maxV4Addresses); - if (maps->d_v4map.fd == -1) { - throw std::runtime_error("Error creating a BPF v4 map of size " + std::to_string(maxV4Addresses) + ": " + stringerror()); - } + uint64_t counter{0}; + uint16_t qtype{0}; +}; - maps->d_v6map.fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(struct KeyV6), sizeof(uint64_t), (int) maxV6Addresses); - if (maps->d_v6map.fd == -1) { - throw std::runtime_error("Error creating a BPF v6 map of size " + std::to_string(maxV6Addresses) + ": " + stringerror()); +struct CounterAndActionValue +{ + uint64_t counter{0}; + BPFFilter::MatchAction action{BPFFilter::MatchAction::Pass}; +}; + +BPFFilter::Map::Map(const BPFFilter::MapConfiguration& config, BPFFilter::MapFormat format): d_config(config) +{ + if (d_config.d_type == BPFFilter::MapType::Filters) { + /* special case, this is a map of eBPF programs */ + d_fd = FDWrapper(bpf_create_map(BPF_MAP_TYPE_PROG_ARRAY, sizeof(uint32_t), sizeof(uint32_t), d_config.d_maxItems)); + if (d_fd.getHandle() == -1) { + throw std::runtime_error("Error creating a BPF program map of size " + std::to_string(d_config.d_maxItems) + ": " + stringerror()); + } } + else { + int keySize = 0; + int valueSize = 0; + if (format == MapFormat::Legacy) { + switch (d_config.d_type) { + case MapType::IPv4: + keySize = sizeof(uint32_t); + valueSize = sizeof(uint64_t); + break; + case MapType::IPv6: + keySize = sizeof(KeyV6); + valueSize = sizeof(uint64_t); + break; + case MapType::QNames: + keySize = sizeof(QNameKey); + valueSize = sizeof(QNameValue); + break; + default: + throw std::runtime_error("Unsupported eBPF map type: " + std::to_string(static_cast(d_config.d_type))); + } + } + else { + switch (d_config.d_type) { + case MapType::IPv4: + keySize = sizeof(uint32_t); + valueSize = sizeof(CounterAndActionValue); + break; + case MapType::IPv6: + keySize = sizeof(KeyV6); + valueSize = sizeof(CounterAndActionValue); + break; + case MapType::QNames: + keySize = sizeof(QNameAndQTypeKey); + valueSize = sizeof(CounterAndActionValue); + break; + default: + throw std::runtime_error("Unsupported eBPF map type: " + std::to_string(static_cast(d_config.d_type))); + } + } + + if (!d_config.d_pinnedPath.empty()) { + /* try to load */ + d_fd = FDWrapper(bpf_load_pinned_map(d_config.d_pinnedPath)); + if (d_fd.getHandle() != -1) { + /* sanity checks: key and value size */ + bpf_check_map_sizes(d_fd.getHandle(), keySize, valueSize); + + if (d_config.d_type == MapType::IPv4) { + uint32_t key = 0; + while (bpf_get_next_key(d_fd.getHandle(), &key, &key) == 0) { + ++d_count; + } + } + else if (d_config.d_type == MapType::IPv6) { + KeyV6 key; + memset(&key, 0, sizeof(key)); + while (bpf_get_next_key(d_fd.getHandle(), &key, &key) == 0) { + ++d_count; + } + } + else if (d_config.d_type == MapType::QNames) { + if (format == MapFormat::Legacy) { + QNameKey key; + memset(&key, 0, sizeof(key)); + while (bpf_get_next_key(d_fd.getHandle(), &key, &key) == 0) { + ++d_count; + } + } + else { + QNameAndQTypeKey key; + memset(&key, 0, sizeof(key)); + while (bpf_get_next_key(d_fd.getHandle(), &key, &key) == 0) { + ++d_count; + } + } + } + } + } - maps->d_qnamemap.fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(struct QNameKey), sizeof(struct QNameValue), (int) maxQNames); - if (maps->d_qnamemap.fd == -1) { - throw std::runtime_error("Error creating a BPF qname map of size " + std::to_string(maxQNames) + ": " + stringerror()); + if (d_fd.getHandle() == -1) { + d_fd = FDWrapper(bpf_create_map(BPF_MAP_TYPE_HASH, keySize, valueSize, static_cast(d_config.d_maxItems))); + if (d_fd.getHandle() == -1) { + throw std::runtime_error("Error creating a BPF map of size " + std::to_string(d_config.d_maxItems) + ": " + stringerror()); + } + + if (!d_config.d_pinnedPath.empty()) { + if (bpf_pin_map(d_fd.getHandle(), d_config.d_pinnedPath) != 0) { + throw std::runtime_error("Unable to pin map to path '" + d_config.d_pinnedPath + "': " + stringerror()); + } + } + } } +} - maps->d_filtermap.fd = bpf_create_map(BPF_MAP_TYPE_PROG_ARRAY, sizeof(uint32_t), sizeof(uint32_t), 1); - if (maps->d_filtermap.fd == -1) { - throw std::runtime_error("Error creating a BPF program map of size 1: " + stringerror()); +static FDWrapper loadProgram(const struct bpf_insn* filter, size_t filterSize) +{ + auto fd = FDWrapper(bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, + filter, + filterSize, + "GPL", + 0)); + if (fd.getHandle() == -1) { + throw std::runtime_error("error loading BPF filter: " + stringerror()); } + return fd; +} - struct bpf_insn main_filter[] = { -#include "bpf-filter.main.ebpf" - }; - d_mainfilter.fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, - main_filter, - sizeof(main_filter), - "GPL", - 0); - if (d_mainfilter.fd == -1) { - throw std::runtime_error("Error loading BPF main filter: " + stringerror()); +BPFFilter::BPFFilter(const BPFFilter::MapConfiguration& v4, const BPFFilter::MapConfiguration& v6, const BPFFilter::MapConfiguration& qnames, BPFFilter::MapFormat format, bool external): d_mapFormat(format), d_external(external) +{ + if (d_mapFormat != BPFFilter::MapFormat::Legacy && !d_external) { + throw std::runtime_error("Unsupported eBPF map format, the current internal implemenation only supports the legacy format"); } - struct bpf_insn qname_filter[] = { + auto maps = d_maps.lock(); + + maps->d_v4 = BPFFilter::Map(v4, d_mapFormat); + maps->d_v6 = BPFFilter::Map(v6, d_mapFormat); + maps->d_qnames = BPFFilter::Map(qnames, d_mapFormat); + if (!external) { + BPFFilter::MapConfiguration filters; + filters.d_maxItems = 1; + filters.d_type = BPFFilter::MapType::Filters; + maps->d_filters = BPFFilter::Map(filters, d_mapFormat); + + const struct bpf_insn main_filter[] = { +#include "bpf-filter.main.ebpf" + }; + + const struct bpf_insn qname_filter[] = { #include "bpf-filter.qname.ebpf" - }; + }; - d_qnamefilter.fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, - qname_filter, - sizeof(qname_filter), - "GPL", - 0); - if (d_qnamefilter.fd == -1) { - throw std::runtime_error("Error loading BPF qname filter: " + stringerror()); - } + try { + d_mainfilter = loadProgram(main_filter, + sizeof(main_filter)); + } + catch (const std::exception& e) { + throw std::runtime_error("Error load the main eBPF filter: " + std::string(e.what())); + } - uint32_t key = 0; - int res = bpf_update_elem(maps->d_filtermap.fd, &key, &d_qnamefilter.fd, BPF_ANY); - if (res != 0) { - throw std::runtime_error("Error updating BPF filters map: " + stringerror()); + try { + d_qnamefilter = loadProgram(qname_filter, + sizeof(qname_filter)); + } + catch (const std::exception& e) { + throw std::runtime_error("Error load the qname eBPF filter: " + std::string(e.what())); + } + + uint32_t key = 0; + int qnamefd = d_qnamefilter.getHandle(); + int res = bpf_update_elem(maps->d_filters.d_fd.getHandle(), &key, &qnamefd, BPF_ANY); + if (res != 0) { + throw std::runtime_error("Error updating BPF filters map: " + stringerror()); + } } } void BPFFilter::addSocket(int sock) { - int res = setsockopt(sock, SOL_SOCKET, SO_ATTACH_BPF, &d_mainfilter.fd, sizeof(d_mainfilter.fd)); + int fd = d_mainfilter.getHandle(); + int res = setsockopt(sock, SOL_SOCKET, SO_ATTACH_BPF, &fd, sizeof(fd)); if (res != 0) { throw std::runtime_error("Error attaching BPF filter to this socket: " + stringerror()); @@ -206,32 +374,37 @@ void BPFFilter::addSocket(int sock) void BPFFilter::removeSocket(int sock) { - int res = setsockopt(sock, SOL_SOCKET, SO_DETACH_BPF, &d_mainfilter.fd, sizeof(d_mainfilter.fd)); + int fd = d_mainfilter.getHandle(); + int res = setsockopt(sock, SOL_SOCKET, SO_DETACH_BPF, &fd, sizeof(fd)); if (res != 0) { throw std::runtime_error("Error detaching BPF filter from this socket: " + stringerror()); } } -void BPFFilter::block(const ComboAddress& addr) +void BPFFilter::block(const ComboAddress& addr, BPFFilter::MatchAction action) { - uint64_t counter = 0; + CounterAndActionValue value; + value.counter = 0; + value.action = action; + int res = 0; if (addr.isIPv4()) { uint32_t key = htonl(addr.sin4.sin_addr.s_addr); auto maps = d_maps.lock(); - if (maps->d_v4Count >= d_maxV4) { + auto& map = maps->d_v4; + if (map.d_count >= map.d_config.d_maxItems) { throw std::runtime_error("Table full when trying to block " + addr.toString()); } - res = bpf_lookup_elem(maps->d_v4map.fd, &key, &counter); + res = bpf_lookup_elem(map.d_fd.getHandle(), &key, &value); if (res != -1) { throw std::runtime_error("Trying to block an already blocked address: " + addr.toString()); } - res = bpf_update_elem(maps->d_v4map.fd, &key, &counter, BPF_NOEXIST); + res = bpf_update_elem(map.d_fd.getHandle(), &key, &value, BPF_NOEXIST); if (res == 0) { - maps->d_v4Count++; + ++map.d_count; } } else if (addr.isIPv6()) { @@ -242,18 +415,19 @@ void BPFFilter::block(const ComboAddress& addr) } auto maps = d_maps.lock(); - if (maps->d_v6Count >= d_maxV6) { + auto& map = maps->d_v6; + if (map.d_count >= map.d_config.d_maxItems) { throw std::runtime_error("Table full when trying to block " + addr.toString()); } - res = bpf_lookup_elem(maps->d_v6map.fd, &key, &counter); + res = bpf_lookup_elem(map.d_fd.getHandle(), &key, &value); if (res != -1) { throw std::runtime_error("Trying to block an already blocked address: " + addr.toString()); } - res = bpf_update_elem(maps->d_v6map.fd, key, &counter, BPF_NOEXIST); + res = bpf_update_elem(map.d_fd.getHandle(), key, &value, BPF_NOEXIST); if (res == 0) { - maps->d_v6Count++; + map.d_count++; } } @@ -268,9 +442,10 @@ void BPFFilter::unblock(const ComboAddress& addr) if (addr.isIPv4()) { uint32_t key = htonl(addr.sin4.sin_addr.s_addr); auto maps = d_maps.lock(); - res = bpf_delete_elem(maps->d_v4map.fd, &key); + auto& map = maps->d_v4; + res = bpf_delete_elem(map.d_fd.getHandle(), &key); if (res == 0) { - maps->d_v4Count--; + --map.d_count; } } else if (addr.isIPv6()) { @@ -281,9 +456,10 @@ void BPFFilter::unblock(const ComboAddress& addr) } auto maps = d_maps.lock(); - res = bpf_delete_elem(maps->d_v6map.fd, key); + auto& map = maps->d_v6; + res = bpf_delete_elem(map.d_fd.getHandle(), key); if (res == 0) { - maps->d_v6Count--; + --map.d_count; } } @@ -292,35 +468,50 @@ void BPFFilter::unblock(const ComboAddress& addr) } } -void BPFFilter::block(const DNSName& qname, uint16_t qtype) +void BPFFilter::block(const DNSName& qname, BPFFilter::MatchAction action, uint16_t qtype) { - struct QNameKey key; - struct QNameValue value; + CounterAndActionValue cadvalue; + QNameValue qvalue; + void* value = nullptr; + + if (d_external) { + memset(&cadvalue, 0, sizeof(cadvalue)); + cadvalue.counter = 0; + cadvalue.action = action; + value = &cadvalue; + } + else { + memset(&qvalue, 0, sizeof(qvalue)); + qvalue.counter = 0; + qvalue.qtype = qtype; + value = &qvalue; + } + + QNameAndQTypeKey key; memset(&key, 0, sizeof(key)); - memset(&value, 0, sizeof(value)); - value.counter = 0; - value.qtype = qtype; std::string keyStr = qname.toDNSStringLC(); if (keyStr.size() > sizeof(key.qname)) { throw std::runtime_error("Invalid QName to block " + qname.toLogString()); } memcpy(key.qname, keyStr.c_str(), keyStr.size()); + key.qtype = qtype; { auto maps = d_maps.lock(); - if (maps->d_qNamesCount >= d_maxQNames) { + auto& map = maps->d_qnames; + if (map.d_count >= map.d_config.d_maxItems) { throw std::runtime_error("Table full when trying to block " + qname.toLogString()); } - int res = bpf_lookup_elem(maps->d_qnamemap.fd, &key, &value); + int res = bpf_lookup_elem(map.d_fd.getHandle(), &key, &value); if (res != -1) { throw std::runtime_error("Trying to block an already blocked qname: " + qname.toLogString()); } - res = bpf_update_elem(maps->d_qnamemap.fd, &key, &value, BPF_NOEXIST); + res = bpf_update_elem(map.d_fd.getHandle(), &key, &value, BPF_NOEXIST); if (res == 0) { - maps->d_qNamesCount++; + ++map.d_count; } if (res != 0) { @@ -331,20 +522,22 @@ void BPFFilter::block(const DNSName& qname, uint16_t qtype) void BPFFilter::unblock(const DNSName& qname, uint16_t qtype) { - struct QNameKey key = { { 0 } }; + QNameAndQTypeKey key; + memset(&key, 0, sizeof(key)); std::string keyStr = qname.toDNSStringLC(); - (void) qtype; if (keyStr.size() > sizeof(key.qname)) { throw std::runtime_error("Invalid QName to block " + qname.toLogString()); } memcpy(key.qname, keyStr.c_str(), keyStr.size()); + key.qtype = qtype; { auto maps = d_maps.lock(); - int res = bpf_delete_elem(maps->d_qnamemap.fd, &key); + auto& map = maps->d_qnames; + int res = bpf_delete_elem(map.d_fd.getHandle(), &key); if (res == 0) { - maps->d_qNamesCount--; + --map.d_count; } else { throw std::runtime_error("Error removing qname address " + qname.toLogString() + ": " + stringerror()); @@ -357,7 +550,7 @@ std::vector > BPFFilter::getAddrStats() std::vector > result; { auto maps = d_maps.lock(); - result.reserve(maps->d_v4Count + maps->d_v6Count); + result.reserve(maps->d_v4.d_count + maps->d_v6.d_count); } sockaddr_in v4Addr; @@ -366,7 +559,7 @@ std::vector > BPFFilter::getAddrStats() uint32_t v4Key = 0; uint32_t nextV4Key; - uint64_t value; + CounterAndActionValue value; uint8_t v6Key[16]; uint8_t nextV6Key[16]; @@ -378,29 +571,37 @@ std::vector > BPFFilter::getAddrStats() memset(&v6Key, 0, sizeof(v6Key)); auto maps = d_maps.lock(); - int res = bpf_get_next_key(maps->d_v4map.fd, &v4Key, &nextV4Key); - while (res == 0) { - v4Key = nextV4Key; - if (bpf_lookup_elem(maps->d_v4map.fd, &v4Key, &value) == 0) { + { + auto& map = maps->d_v4; + int res = bpf_get_next_key(map.d_fd.getHandle(), &v4Key, &nextV4Key); + + while (res == 0) { + v4Key = nextV4Key; + if (bpf_lookup_elem(map.d_fd.getHandle(), &v4Key, &value) == 0) { v4Addr.sin_addr.s_addr = ntohl(v4Key); - result.push_back(make_pair(ComboAddress(&v4Addr), value)); - } + result.emplace_back(ComboAddress(&v4Addr), value.counter); + } - res = bpf_get_next_key(maps->d_v4map.fd, &v4Key, &nextV4Key); + res = bpf_get_next_key(map.d_fd.getHandle(), &v4Key, &nextV4Key); + } } - res = bpf_get_next_key(maps->d_v6map.fd, &v6Key, &nextV6Key); + { + auto& map = maps->d_v6; + int res = bpf_get_next_key(map.d_fd.getHandle(), &v6Key, &nextV6Key); - while (res == 0) { - if (bpf_lookup_elem(maps->d_v6map.fd, &nextV6Key, &value) == 0) { - memcpy(&v6Addr.sin6_addr.s6_addr, &nextV6Key, sizeof(nextV6Key)); + while (res == 0) { + if (bpf_lookup_elem(map.d_fd.getHandle(), &nextV6Key, &value) == 0) { + memcpy(&v6Addr.sin6_addr.s6_addr, &nextV6Key, sizeof(nextV6Key)); - result.push_back(make_pair(ComboAddress(&v6Addr), value)); - } + result.emplace_back(ComboAddress(&v6Addr), value.counter); + } - res = bpf_get_next_key(maps->d_v6map.fd, &nextV6Key, &nextV6Key); + res = bpf_get_next_key(map.d_fd.getHandle(), &nextV6Key, &nextV6Key); + } } + return result; } @@ -408,35 +609,62 @@ std::vector > BPFFilter::getQNameStats() { std::vector > result; - struct QNameKey key = { { 0 } }; - struct QNameKey nextKey = { { 0 } }; - struct QNameValue value; + if (d_mapFormat == MapFormat::Legacy) { + QNameKey key = { { 0 } }; + QNameKey nextKey = { { 0 } }; + QNameValue value; - auto maps = d_maps.lock(); - result.reserve(maps->d_qNamesCount); - int res = bpf_get_next_key(maps->d_qnamemap.fd, &key, &nextKey); + auto maps = d_maps.lock(); + auto& map = maps->d_qnames; + result.reserve(map.d_count); + int res = bpf_get_next_key(map.d_fd.getHandle(), &key, &nextKey); + + while (res == 0) { + if (bpf_lookup_elem(map.d_fd.getHandle(), &nextKey, &value) == 0) { + nextKey.qname[sizeof(nextKey.qname) - 1 ] = '\0'; + result.push_back(std::make_tuple(DNSName(reinterpret_cast(nextKey.qname), sizeof(nextKey.qname), 0, false), value.qtype, value.counter)); + } - while (res == 0) { - if (bpf_lookup_elem(maps->d_qnamemap.fd, &nextKey, &value) == 0) { - nextKey.qname[sizeof(nextKey.qname) - 1 ] = '\0'; - result.push_back(std::make_tuple(DNSName((const char*) nextKey.qname, sizeof(nextKey.qname), 0, false), value.qtype, value.counter)); + res = bpf_get_next_key(map.d_fd.getHandle(), &nextKey, &nextKey); } + } + else { + QNameAndQTypeKey key; + QNameAndQTypeKey nextKey; + memset(&key, 0, sizeof(key)); + memset(&nextKey, 0, sizeof(nextKey)); + CounterAndActionValue value; + + auto maps = d_maps.lock(); + auto& map = maps->d_qnames; + result.reserve(map.d_count); + int res = bpf_get_next_key(map.d_fd.getHandle(), &key, &nextKey); + + while (res == 0) { + if (bpf_lookup_elem(map.d_fd.getHandle(), &nextKey, &value) == 0) { + nextKey.qname[sizeof(nextKey.qname) - 1 ] = '\0'; + result.push_back(std::make_tuple(DNSName(reinterpret_cast(nextKey.qname), sizeof(nextKey.qname), 0, false), key.qtype, value.counter)); + } - res = bpf_get_next_key(maps->d_qnamemap.fd, &nextKey, &nextKey); + res = bpf_get_next_key(map.d_fd.getHandle(), &nextKey, &nextKey); + } } + return result; } uint64_t BPFFilter::getHits(const ComboAddress& requestor) { - uint64_t counter = 0; + CounterAndActionValue counter; + if (requestor.isIPv4()) { uint32_t key = htonl(requestor.sin4.sin_addr.s_addr); auto maps = d_maps.lock(); - int res = bpf_lookup_elem(maps->d_v4map.fd, &key, &counter); + auto& map = maps->d_v4; + int res = bpf_lookup_elem(map.d_fd.getHandle(), &key, &counter); if (res == 0) { - return counter; + return counter.counter; } } else if (requestor.isIPv6()) { @@ -447,9 +675,10 @@ uint64_t BPFFilter::getHits(const ComboAddress& requestor) } auto maps = d_maps.lock(); - int res = bpf_lookup_elem(maps->d_v6map.fd, &key, &counter); + auto& map = maps->d_v6; + int res = bpf_lookup_elem(map.d_fd.getHandle(), &key, &counter); if (res == 0) { - return counter; + return counter.counter; } } @@ -458,48 +687,37 @@ uint64_t BPFFilter::getHits(const ComboAddress& requestor) #else -BPFFilter::BPFFilter(uint32_t maxV4Addresses, uint32_t maxV6Addresses, uint32_t maxQNames) +BPFFilter::BPFFilter(const BPFFilter::MapConfiguration&, const BPFFilter::MapConfiguration&, const BPFFilter::MapConfiguration&, BPFFilter::MapFormat, bool) { - (void) maxV4Addresses; - (void) maxV6Addresses; - (void) maxQNames; } -void BPFFilter::addSocket(int sock) +void BPFFilter::addSocket(int) { - (void) sock; throw std::runtime_error("eBPF support not enabled"); } -void BPFFilter::removeSocket(int sock) +void BPFFilter::removeSocket(int) { - (void) sock; throw std::runtime_error("eBPF support not enabled"); } -void BPFFilter::block(const ComboAddress& addr) +void BPFFilter::block(const ComboAddress&, BPFFilter::MatchAction) { - (void) addr; throw std::runtime_error("eBPF support not enabled"); } -void BPFFilter::unblock(const ComboAddress& addr) +void BPFFilter::unblock(const ComboAddress&) { - (void) addr; throw std::runtime_error("eBPF support not enabled"); } -void BPFFilter::block(const DNSName& qname, uint16_t qtype) +void BPFFilter::block(const DNSName&, BPFFilter::MatchAction, uint16_t) { - (void) qname; - (void) qtype; throw std::runtime_error("eBPF support not enabled"); } -void BPFFilter::unblock(const DNSName& qname, uint16_t qtype) +void BPFFilter::unblock(const DNSName&, uint16_t) { - (void) qname; - (void) qtype; throw std::runtime_error("eBPF support not enabled"); } @@ -515,9 +733,27 @@ std::vector > BPFFilter::getQNameStats() return result; } -uint64_t BPFFilter::getHits(const ComboAddress& requestor) +uint64_t BPFFilter::getHits(const ComboAddress&) { - (void) requestor; return 0; } #endif /* HAVE_EBPF */ + +bool BPFFilter::supportsMatchAction(MatchAction action) const +{ +#ifdef HAVE_EBPF + if (action == BPFFilter::MatchAction::Drop) { + return true; + } + return d_mapFormat == BPFFilter::MapFormat::WithActions; +#endif /* HAVE_EBPF */ + return false; +} + +bool BPFFilter::isExternal() const +{ +#ifdef HAVE_EBPF + return d_external; +#endif /* HAVE_EBPF */ + return false; +} diff --git a/pdns/bpf-filter.hh b/pdns/bpf-filter.hh index abf805e4c2cc..a8e2f5c83430 100644 --- a/pdns/bpf-filter.hh +++ b/pdns/bpf-filter.hh @@ -28,44 +28,97 @@ class BPFFilter { public: - BPFFilter(uint32_t maxV4Addresses, uint32_t maxV6Addresses, uint32_t maxQNames); + enum class MapType : uint8_t { + IPv4, + IPv6, + QNames, + Filters + }; + + enum class MapFormat : uint8_t { + Legacy = 0, + WithActions = 1 + }; + + enum class MatchAction : uint8_t { + Pass = 0, + Drop = 1, + Truncate = 2 + }; + + struct MapConfiguration + { + std::string d_pinnedPath; + uint32_t d_maxItems{0}; + MapType d_type; + }; + + + BPFFilter(const BPFFilter::MapConfiguration& v4, const BPFFilter::MapConfiguration& v6, const BPFFilter::MapConfiguration& qnames, BPFFilter::MapFormat format, bool external); + BPFFilter(const BPFFilter&) = delete; + BPFFilter(BPFFilter&&) = delete; + BPFFilter& operator=(const BPFFilter&) = delete; + BPFFilter& operator=(BPFFilter&&) = delete; + void addSocket(int sock); void removeSocket(int sock); - void block(const ComboAddress& addr); - void block(const DNSName& qname, uint16_t qtype=255); + void block(const ComboAddress& addr, MatchAction action); + void block(const DNSName& qname, MatchAction action, uint16_t qtype=255); void unblock(const ComboAddress& addr); void unblock(const DNSName& qname, uint16_t qtype=255); + std::vector > getAddrStats(); std::vector > getQNameStats(); + uint64_t getHits(const ComboAddress& requestor); + bool supportsMatchAction(MatchAction action) const; + bool isExternal() const; + private: #ifdef HAVE_EBPF - struct FDWrapper + struct Map { - ~FDWrapper() + Map() { - if (fd != -1) { - close(fd); - } } - int fd{-1}; + Map(const MapConfiguration&, MapFormat); + MapConfiguration d_config; + uint32_t d_count{0}; + FDWrapper d_fd; }; + struct Maps { - FDWrapper d_v4map; - FDWrapper d_v6map; - FDWrapper d_qnamemap; - FDWrapper d_filtermap; - uint32_t d_v4Count{0}; - uint32_t d_v6Count{0}; - uint32_t d_qNamesCount{0}; + Map d_v4; + Map d_v6; + Map d_qnames; + /* The qname filter program held in d_qnamefilter is + stored in an eBPF map, so we can call it from the + main filter. This is the only entry in that map. */ + Map d_filters; }; + LockGuarded d_maps; + + /* main eBPF program */ FDWrapper d_mainfilter; + /* qname filtering program */ FDWrapper d_qnamefilter; - uint32_t d_maxV4; - uint32_t d_maxV6; - uint32_t d_maxQNames; + + /* whether the maps are in the 'old' format, which we need + to keep to prevent going over the 4k instructions per eBPF + program limit in kernels < 5.2, as well as the complexity limit: + - 32k in Linux 3.18 + - 64k in Linux 4.7 + - 96k in Linux 4.12 + - 128k in Linux 4.14, + - 1M in Linux 5.2 */ + MapFormat d_mapFormat; + + /* whether the filter is internal, using our own eBPF programs, + or external where we only update the maps but the filtering is + done by an external program. */ + bool d_external; #endif /* HAVE_EBPF */ }; diff --git a/pdns/bpf-filter.main.ebpf b/pdns/bpf-filter.main.ebpf index a8ffa37ad1f1..4b42a2ec6b71 100644 --- a/pdns/bpf-filter.main.ebpf +++ b/pdns/bpf-filter.main.ebpf @@ -6,7 +6,7 @@ BPF_JMP_IMM(BPF_JEQ,BPF_REG_1,ntohs(0x86dd),11), BPF_JMP_IMM(BPF_JNE,BPF_REG_1,ntohs(0x0800),109), BPF_LD_ABS(BPF_W,-2097126), BPF_STX_MEM(BPF_W,BPF_REG_10,BPF_REG_0,-256), -BPF_LD_MAP_FD(BPF_REG_1,maps->d_v4map.fd), +BPF_LD_MAP_FD(BPF_REG_1,maps->d_v4.d_fd.getHandle()), BPF_MOV64_REG(BPF_REG_2,BPF_REG_10), BPF_ALU64_IMM(BPF_ADD,BPF_REG_2,-256), BPF_RAW_INSN(BPF_JMP|BPF_CALL,0,0,0,BPF_FUNC_map_lookup_elem), @@ -45,7 +45,7 @@ BPF_LD_ABS(BPF_B,-2097116), BPF_STX_MEM(BPF_B,BPF_REG_10,BPF_REG_0,-242), BPF_LD_ABS(BPF_B,-2097115), BPF_STX_MEM(BPF_B,BPF_REG_10,BPF_REG_0,-241), -BPF_LD_MAP_FD(BPF_REG_1,maps->d_v6map.fd), +BPF_LD_MAP_FD(BPF_REG_1,maps->d_v6.d_fd.getHandle()), BPF_MOV64_REG(BPF_REG_2,BPF_REG_10), BPF_ALU64_IMM(BPF_ADD,BPF_REG_2,-256), BPF_RAW_INSN(BPF_JMP|BPF_CALL,0,0,0,BPF_FUNC_map_lookup_elem), @@ -97,7 +97,7 @@ BPF_JMP_IMM(BPF_JGT,BPF_REG_8,63,17), BPF_JMP_IMM(BPF_JNE,BPF_REG_8,0,18), BPF_LD_ABS(BPF_H,21), BPF_MOV64_REG(BPF_REG_6,BPF_REG_0), -BPF_LD_MAP_FD(BPF_REG_1,maps->d_qnamemap.fd), +BPF_LD_MAP_FD(BPF_REG_1,maps->d_qnames.d_fd.getHandle()), BPF_MOV64_REG(BPF_REG_2,BPF_REG_10), BPF_ALU64_IMM(BPF_ADD,BPF_REG_2,-256), BPF_RAW_INSN(BPF_JMP|BPF_CALL,0,0,0,BPF_FUNC_map_lookup_elem), @@ -128,7 +128,7 @@ BPF_ALU64_IMM(BPF_ADD,BPF_REG_8,-1), BPF_STX_MEM(BPF_W,BPF_REG_6,BPF_REG_8,60), BPF_ALU64_IMM(BPF_AND,BPF_REG_1,255), BPF_STX_MEM(BPF_W,BPF_REG_6,BPF_REG_1,56), -BPF_LD_MAP_FD(BPF_REG_2,maps->d_filtermap.fd), +BPF_LD_MAP_FD(BPF_REG_2,maps->d_filters.d_fd.getHandle()), BPF_MOV64_REG(BPF_REG_1,BPF_REG_6), BPF_MOV64_IMM(BPF_REG_3,0), BPF_RAW_INSN(BPF_JMP|BPF_CALL,0,0,0,BPF_FUNC_tail_call), diff --git a/pdns/bpf-filter.qname.ebpf b/pdns/bpf-filter.qname.ebpf index febb7b862c47..c7d6094c0872 100644 --- a/pdns/bpf-filter.qname.ebpf +++ b/pdns/bpf-filter.qname.ebpf @@ -4078,7 +4078,7 @@ BPF_MOV64_IMM(BPF_REG_9,255), BPF_ALU64_REG(BPF_ADD,BPF_REG_9,BPF_REG_7), BPF_RAW_INSN(BPF_LD|BPF_IND|BPF_H,BPF_REG_0,BPF_REG_9,0,0), BPF_MOV64_REG(BPF_REG_6,BPF_REG_0), -BPF_LD_MAP_FD(BPF_REG_1,maps->d_qnamemap.fd), +BPF_LD_MAP_FD(BPF_REG_1,maps->d_qnames.d_fd.getHandle()), BPF_MOV64_REG(BPF_REG_2,BPF_REG_10), BPF_ALU64_IMM(BPF_ADD,BPF_REG_2,-256), BPF_RAW_INSN(BPF_JMP|BPF_CALL,0,0,0,BPF_FUNC_map_lookup_elem), diff --git a/pdns/cachecleaner.hh b/pdns/cachecleaner.hh index fd164fa6978a..f3af11af7765 100644 --- a/pdns/cachecleaner.hh +++ b/pdns/cachecleaner.hh @@ -102,12 +102,12 @@ template uint64_t pruneLockedCollectionsVector(std::vec time_t now = time(nullptr); for(auto& mc : maps) { - WriteLock wl(&mc.d_mut); + auto map = mc.d_map.write_lock(); - uint64_t lookAt = (mc.d_map.size() + 9) / 10; // Look at 10% of this shard + uint64_t lookAt = (map->size() + 9) / 10; // Look at 10% of this shard uint64_t erased = 0; - auto& sidx = boost::multi_index::get(mc.d_map); + auto& sidx = boost::multi_index::get(*map); for(auto i = sidx.begin(); i != sidx.end() && lookAt > 0; lookAt--) { if(i->ttd < now) { i = sidx.erase(i); @@ -203,9 +203,9 @@ template uint64_t purgeLockedCollectionsVector(std::vector& maps uint64_t delcount=0; for(auto& mc : maps) { - WriteLock wl(&mc.d_mut); - delcount += mc.d_map.size(); - mc.d_map.clear(); + auto map = mc.d_map.write_lock(); + delcount += map->size(); + map->clear(); } return delcount; @@ -218,8 +218,8 @@ template uint64_t purgeLockedCollectionsVector(std::vec prefix.resize(prefix.size()-1); DNSName dprefix(prefix); for(auto& mc : maps) { - WriteLock wl(&mc.d_mut); - auto& idx = boost::multi_index::get(mc.d_map); + auto map = mc.d_map.write_lock(); + auto& idx = boost::multi_index::get(*map); auto iter = idx.lower_bound(dprefix); auto start = iter; @@ -238,8 +238,8 @@ template uint64_t purgeLockedCollectionsVector(std::vec template uint64_t purgeExactLockedCollection(T& mc, const DNSName& qname) { uint64_t delcount=0; - WriteLock wl(&mc.d_mut); - auto& idx = boost::multi_index::get(mc.d_map); + auto map = mc.d_map.write_lock(); + auto& idx = boost::multi_index::get(*map); auto range = idx.equal_range(qname); if(range.first != range.second) { delcount += distance(range.first, range.second); diff --git a/pdns/calidns.cc b/pdns/calidns.cc index e8dac6eb6d92..973e5309538e 100644 --- a/pdns/calidns.cc +++ b/pdns/calidns.cc @@ -399,14 +399,14 @@ try if(!subnet.empty() || !ecsRange.empty()) { EDNSSubnetOpts opt; opt.source = Netmask(subnet.empty() ? "0.0.0.0/32" : subnet); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); } if(!ednsOptions.empty() || pw.getHeader()->id % 2) { pw.addOpt(1500, 0, EDNSOpts::DNSSECOK, ednsOptions); pw.commit(); } - unknown.emplace_back(std::make_shared>(packet)); + unknown.push_back(std::make_shared>(packet)); } shuffle(unknown.begin(), unknown.end(), pdns::dns_random_engine()); diff --git a/pdns/cdb.cc b/pdns/cdb.cc index 5dd391539f1a..55ade934e521 100644 --- a/pdns/cdb.cc +++ b/pdns/cdb.cc @@ -123,7 +123,7 @@ bool CDB::readNext(pair &value) { throw std::runtime_error("Error while reading value for key '" + key + "' from CDB database: " + std::to_string(ret)); } - value = make_pair(std::move(key), std::move(val)); + value = {std::move(key), std::move(val)}; return true; } diff --git a/pdns/comfun.cc b/pdns/comfun.cc index 9753e9f0d200..01a1fff2cc3f 100644 --- a/pdns/comfun.cc +++ b/pdns/comfun.cc @@ -68,9 +68,13 @@ struct SendReceive SendReceive(map& res) : d_res(res) { d_socket = socket(AF_INET, SOCK_DGRAM, 0); + if (d_socket < 0) + throw runtime_error("socket(AF_INET, SOCK_DGRAM, 0) failed: " + stringerror()); int val=1; - setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); + if (setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) != 0) + cerr<<"Failed to turn on SO_REUSEADDR: " << stringerror() << endl; + for(unsigned int id =0 ; id < std::numeric_limits::max(); ++id) d_idqueue.push_back(id); } @@ -198,8 +202,11 @@ struct SendReceiveRes { d_socket = socket(AF_INET, SOCK_DGRAM, 0); int val=1; - setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)); - connect(d_socket, (struct sockaddr*)&remote, remote.getSocklen()); + + if (setsockopt(d_socket, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) != 0) + cerr<<"Failed to turn on SO_REUSEADDR: " << stringerror() << endl; + if (connect(d_socket, (struct sockaddr*)&remote, remote.getSocklen()) != 0) + throw DNSException("connect(" + remote.toStringWithPort() + ") failed: " + stringerror()); for(unsigned int id =0 ; id < std::numeric_limits::max(); ++id) d_idqueue.push_back(id); } @@ -299,7 +306,7 @@ struct SendReceiveRes }; -void printStats() +static void printStats() { while(!g_stop) { sleep(1); @@ -309,7 +316,7 @@ void printStats() cerr<<"\n"; } -int parseZone(const std::string& str, unsigned int limit) +static int parseZone(const std::string& str, unsigned int limit) { ZoneParserTNG zpt(str); zpt.disableGenerate(); @@ -353,7 +360,7 @@ int parseZone(const std::string& str, unsigned int limit) return 0; } -int resolveNS(const std::string& fname) +static int resolveNS(const std::string& fname) { string line; ifstream needres(fname); @@ -385,7 +392,7 @@ int resolveNS(const std::string& fname) return EXIT_SUCCESS; } -void readRESNames(const std::string& fname, map>& addrs) +static void readRESNames(const std::string& fname, map>& addrs) { ifstream ifs(fname); if(!ifs) @@ -422,7 +429,7 @@ try } else if(mode=="scan-ns") { ifstream ns(string(argv[2])+".nameservers"); - g_powerdns = std::unique_ptr(new ofstream(string(argv[2])+".powerdns")); + g_powerdns = std::make_unique(string(argv[2]) + ".powerdns"); string line; int count=0; vector parts; diff --git a/pdns/common_startup.cc b/pdns/common_startup.cc index afc9b0b685d5..39dab491603b 100644 --- a/pdns/common_startup.cc +++ b/pdns/common_startup.cc @@ -64,6 +64,8 @@ double avg_latency{0.0}; unique_ptr TN; static vector g_distributors; vector > g_udpReceivers; +NetmaskGroup g_proxyProtocolACL; +size_t g_proxyProtocolMaximumSize; ArgvMap &arg() { @@ -93,6 +95,8 @@ void declareArguments() ::arg().setSwitch("dnsupdate","Enable/Disable DNS update (RFC2136) support. Default is no.")="no"; ::arg().setSwitch("write-pid","Write a PID file")="yes"; ::arg().set("allow-dnsupdate-from","A global setting to allow DNS updates from these IP ranges.")="127.0.0.0/8,::1"; + ::arg().set("proxy-protocol-from", "A Proxy Protocol header is only allowed from these subnets, and is mandatory then too.")=""; + ::arg().set("proxy-protocol-maximum-size", "The maximum size of a proxy protocol payload, including the TLV values")="512"; ::arg().setSwitch("send-signed-notify", "Send TSIG secured NOTIFY if TSIG key is configured for a zone") = "yes"; ::arg().set("allow-unsigned-notify", "Allow unsigned notifications for TSIG secured zones") = "yes"; //FIXME: change to 'no' later ::arg().set("allow-unsigned-supermaster", "Allow supermasters to create zones without TSIG signed NOTIFY")="yes"; @@ -161,6 +165,8 @@ void declareArguments() ::arg().setSwitch("any-to-tcp","Answer ANY queries with tc=1, shunting to TCP")="yes"; ::arg().setSwitch("edns-subnet-processing","If we should act on EDNS Subnet options")="no"; + ::arg().set("edns-cookie-secret", "When set, set a server cookie when responding to a query with a Client cookie (in hex)")=""; + ::arg().setSwitch("webserver","Start a webserver for monitoring (api=yes also enables the HTTP listener)")="no"; ::arg().setSwitch("webserver-print-arguments","If the webserver should print arguments")="no"; ::arg().set("webserver-address","IP Address of webserver/API to listen on")="127.0.0.1"; @@ -169,6 +175,7 @@ void declareArguments() ::arg().set("webserver-allow-from","Webserver/API access is only allowed from these subnets")="127.0.0.1,::1"; ::arg().set("webserver-loglevel", "Amount of logging in the webserver (none, normal, detailed)") = "normal"; ::arg().set("webserver-max-bodysize","Webserver/API maximum request/response body size in megabytes")="2"; + ::arg().setSwitch("webserver-hash-plaintext-credentials","Whether to hash passwords and api keys supplied in plaintext, to prevent keeping the plaintext version in memory at runtime")="no"; ::arg().setSwitch("query-logging","Hint backends that queries should be logged")="no"; @@ -247,6 +254,7 @@ void declareArguments() ::arg().set("tcp-fast-open", "Enable TCP Fast Open support on the listening sockets, using the supplied numerical value as the queue size")="0"; ::arg().set("max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0"; + ::arg().set("max-include-depth", "Maximum number of nested $INCLUDE directives while processing a zone file")="20"; ::arg().setSwitch("upgrade-unknown-types","Transparently upgrade known TYPExxx records. Recommended to keep off, except for PowerDNS upgrades until data sources are cleaned up")="no"; ::arg().setSwitch("svc-autohints", "Transparently fill ipv6hint=auto ipv4hint=auto SVC params with AAAA/A records for the target name of the record (if within the same zone)")="no"; @@ -311,6 +319,7 @@ void declareStats() { S.declare("udp-queries","Number of UDP queries received"); S.declare("udp-do-queries","Number of UDP queries received with DO bit"); + S.declare("udp-cookie-queries", "Number of UDP queries received with the COOKIE EDNS option"); S.declare("udp-answers","Number of answers sent out over UDP"); S.declare("udp-answers-bytes","Total size of answers sent out over UDP"); S.declare("udp4-answers-bytes","Total size of answers sent out over UDPv4"); @@ -329,6 +338,7 @@ void declareStats() S.declare("corrupt-packets","Number of corrupt packets received"); S.declare("signatures", "Number of DNSSEC signatures made"); S.declare("tcp-queries","Number of TCP queries received"); + S.declare("tcp-cookie-queries","Number of TCP queries received with the COOKIE option"); S.declare("tcp-answers","Number of answers sent out over TCP"); S.declare("tcp-answers-bytes","Total size of answers sent out over TCP"); S.declare("tcp4-answers-bytes","Total size of answers sent out over TCPv4"); @@ -360,6 +370,12 @@ void declareStats() S.declare("udp-sndbuf-errors", "UDP 'sndbuf' errors", udpErrorStats, StatType::counter); S.declare("udp-noport-errors", "UDP 'noport' errors", udpErrorStats, StatType::counter); S.declare("udp-in-errors", "UDP 'in' errors", udpErrorStats, StatType::counter); + S.declare("udp-in-csum-errors", "UDP 'in checksum' errors", udpErrorStats, StatType::counter); + S.declare("udp6-in-errors", "UDP 'in' errors over IPv6", udp6ErrorStats, StatType::counter); + S.declare("udp6-recvbuf-errors", "UDP 'recvbuf' errors over IPv6", udp6ErrorStats, StatType::counter); + S.declare("udp6-sndbuf-errors", "UDP 'sndbuf' errors over IPv6", udp6ErrorStats, StatType::counter); + S.declare("udp6-noport-errors", "UDP 'noport' errors over IPv6", udp6ErrorStats, StatType::counter); + S.declare("udp6-in-csum-errors", "UDP 'in checksum' errors over IPv6", udp6ErrorStats, StatType::counter); #endif S.declare("sys-msec", "Number of msec spent in system time", getSysUserTimeMsec, StatType::counter); @@ -405,11 +421,16 @@ static void sendout(std::unique_ptr& a) { if(!a) return; - - N->send(*a); - int diff=a->d_dt.udiff(); - avg_latency=0.999*avg_latency+0.001*diff; + try { + N->send(*a); + + int diff=a->d_dt.udiff(); + avg_latency=0.999*avg_latency+0.001*diff; + } + catch (const std::exception& e) { + g_log< NS; std::string buffer; - buffer.resize(DNSPacket::s_udpTruncationThreshold); // If we have SO_REUSEPORT then create a new port for all receiver threads // other than the first one. @@ -449,76 +470,86 @@ try } for(;;) { - if(!NS->receive(question, buffer)) { // receive a packet inline - continue; // packet was broken, try again - } + try { + if (g_proxyProtocolACL.empty()) { + buffer.resize(DNSPacket::s_udpTruncationThreshold); + } + else { + buffer.resize(DNSPacket::s_udpTruncationThreshold + g_proxyProtocolMaximumSize); + } - numreceived++; + if(!NS->receive(question, buffer)) { // receive a packet inline + continue; // packet was broken, try again + } - if(question.d_remote.getSocklen()==sizeof(sockaddr_in)) - numreceived4++; - else - numreceived6++; + numreceived++; - if(question.d_dnssecOk) - numreceiveddo++; + if(question.d_remote.getSocklen()==sizeof(sockaddr_in)) + numreceived4++; + else + numreceived6++; - if(question.d.qr) - continue; + if(question.d_dnssecOk) + numreceiveddo++; - S.ringAccount("queries", question.qdomain, question.qtype); - S.ringAccount("remotes", question.d_remote); - if(logDNSQueries) { - string remote; - if(question.hasEDNSSubnet()) - remote = question.getRemote().toString() + "<-" + question.getRealRemote().toString(); - else - remote = question.getRemote().toString(); - g_log << Logger::Notice<<"Remote "<< remote <<" wants '" << question.qdomain<<"|"<send(cached); // answer it then inlined - diff=question.d_dt.udiff(); - avg_latency=0.999*avg_latency+0.001*diff; // 'EWMA' + if(question.d.qr) continue; + + S.ringAccount("queries", question.qdomain, question.qtype); + S.ringAccount("remotes", question.d_remote); + if(logDNSQueries) { + g_log << Logger::Notice<<"Remote "<< question.getRemoteString() <<" wants '" << question.qdomain<<"|"<isOverloaded()) { - if(logDNSQueries) - g_log<<": Dropped query, backends are overloaded"<send(cached); // answer it then inlined + diff=question.d_dt.udiff(); + avg_latency=0.999*avg_latency+0.001*diff; // 'EWMA' + continue; + } + } - if (logDNSQueries) { - if (PC.enabled()) { - g_log<<": packetcache MISS"<isOverloaded()) { + if(logDNSQueries) + g_log<<": Dropped query, backends are overloaded"<question(question, &sendout); // otherwise, give to the distributor + if (logDNSQueries) { + if (PC.enabled()) { + g_log<<": packetcache MISS"<question(question, &sendout); // otherwise, give to the distributor + } + catch(DistributorFatal& df) { // when this happens, we have leaked loads of memory. Bailing out time. + _exit(1); + } } - catch(DistributorFatal& df) { // when this happens, we have leaked loads of memory. Bailing out time. - _exit(1); + catch (const std::exception& e) { + g_log<(new DNSProxy(::arg()["resolver"])); + DP = std::make_unique(::arg()["resolver"]); DP->go(); } diff --git a/pdns/common_startup.hh b/pdns/common_startup.hh index c2e46ad60013..849bf243579f 100644 --- a/pdns/common_startup.hh +++ b/pdns/common_startup.hh @@ -53,6 +53,8 @@ extern int isGuarded( char ** ); void carbonDumpThread(); extern bool g_anyToTcp; extern bool g_8bitDNS; +extern NetmaskGroup g_proxyProtocolACL; +extern size_t g_proxyProtocolMaximumSize; #ifdef HAVE_LUA_RECORDS extern bool g_doLuaRecord; extern bool g_LuaRecordSharedState; diff --git a/pdns/communicator.cc b/pdns/communicator.cc index cb48445b183f..fd44af19fe92 100644 --- a/pdns/communicator.cc +++ b/pdns/communicator.cc @@ -47,16 +47,17 @@ void CommunicatorClass::retrievalLoopThread() d_suck_sem.wait(); SuckRequest sr; { - std::lock_guard l(d_lock); - if(d_suckdomains.empty()) + auto data = d_data.lock(); + if (data->d_suckdomains.empty()) { continue; + } - auto firstItem = d_suckdomains.begin(); + auto firstItem = data->d_suckdomains.begin(); sr=*firstItem; - d_suckdomains.erase(firstItem); - if (d_suckdomains.empty()) { - d_sorthelper = 0; + data->d_suckdomains.erase(firstItem); + if (data->d_suckdomains.empty()) { + data->d_sorthelper = 0; } } suck(sr.domain, sr.master, sr.force); @@ -141,9 +142,10 @@ void CommunicatorClass::mainloop() bool extraSlaveRefresh = false; Utility::sleep(1); { - std::lock_guard l(d_lock); - if (d_tocheck.size()) + auto data = d_data.lock(); + if (data->d_tocheck.size()) { extraSlaveRefresh = true; + } } if (extraSlaveRefresh) slaveRefresh(&P); diff --git a/pdns/communicator.hh b/pdns/communicator.hh index 5c993e816a26..0ffeef30a7aa 100644 --- a/pdns/communicator.hh +++ b/pdns/communicator.hh @@ -157,7 +157,6 @@ public: d_nsock4 = -1; d_nsock6 = -1; d_preventSelfNotification = false; - d_sorthelper = 0; } time_t doNotifications(PacketHandler *P); void go(); @@ -180,42 +179,48 @@ private: void makeNotifySockets(); void queueNotifyDomain(const DomainInfo& di, UeberBackend* B); int d_nsock4, d_nsock6; - map,time_t>d_holes; - std::mutex d_holelock; + LockGuarded,time_t>> d_holes; + void suck(const DNSName &domain, const ComboAddress& remote, bool force=false); void ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, const ComboAddress& laddr, const ComboAddress& remote, std::unique_ptr& pdl, ZoneStatus& zs, vector* axfr); void slaveRefresh(PacketHandler *P); void masterUpdateCheck(PacketHandler *P); - std::mutex d_lock; - - uint64_t d_sorthelper; - UniQueue d_suckdomains; - set d_inprogress; - + Semaphore d_suck_sem; Semaphore d_any_sem; - time_t d_tickinterval; - set d_tocheck; - struct cmp { - bool operator()(const DNSPacket& a, const DNSPacket& b) const { - return a.qdomain < b.qdomain; - }; - }; - - std::set d_potentialsupermasters; set d_alsoNotify; - NotificationQueue d_nq; NetmaskGroup d_onlyNotify; + NotificationQueue d_nq; + + time_t d_tickinterval; bool d_masterschanged, d_slaveschanged; bool d_preventSelfNotification; - // Used to keep some state on domains that failed their freshness checks. - // uint64_t == counter of the number of failures (increased by 1 every consecutive slave-cycle-interval that the domain fails) - // time_t == wait at least until this time before attempting a new check - map > d_failedSlaveRefresh; + struct Data + { + uint64_t d_sorthelper{0}; + UniQueue d_suckdomains; + set d_inprogress; + + set d_tocheck; + struct cmp { + bool operator()(const DNSPacket& a, const DNSPacket& b) const { + return a.qdomain < b.qdomain; + }; + }; + + std::set d_potentialsupermasters; + + // Used to keep some state on domains that failed their freshness checks. + // uint64_t == counter of the number of failures (increased by 1 every consecutive slave-cycle-interval that the domain fails) + // time_t == wait at least until this time before attempting a new check + map > d_failedSlaveRefresh; + }; + + LockGuarded d_data; struct RemoveSentinel { @@ -225,8 +230,7 @@ private: ~RemoveSentinel() { try { - std::lock_guard l(d_cc->d_lock); - d_cc->d_inprogress.erase(d_dn); + d_cc->d_data.lock()->d_inprogress.erase(d_dn); } catch(...) { } diff --git a/pdns/credentials.cc b/pdns/credentials.cc new file mode 100644 index 000000000000..d058a948ac1c --- /dev/null +++ b/pdns/credentials.cc @@ -0,0 +1,477 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "config.h" + +#include +#include + +#ifdef HAVE_LIBSODIUM +#include +#endif + +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT +#include +#include +#include +#include +#endif + +#include +#include +#include + +#include "base64.hh" +#include "credentials.hh" +#include "misc.hh" + +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT +static size_t const pwhash_max_size = 128U; /* maximum size of the output */ +static size_t const pwhash_output_size = 32U; /* size of the hashed output (before base64 encoding) */ +static unsigned int const pwhash_salt_size = 16U; /* size of the salt (before base64 encoding */ +static uint64_t const pwhash_max_work_factor = 32768U; /* max N for interactive login purposes */ + +/* PHC string format, storing N as log2(N) as done by passlib. + for now we only support one algo but we might have to change that later */ +static std::string const pwhash_prefix = "$scrypt$"; +static size_t const pwhash_prefix_size = pwhash_prefix.size(); +#endif + +uint64_t const CredentialsHolder::s_defaultWorkFactor{1024U}; /* N */ +uint64_t const CredentialsHolder::s_defaultParallelFactor{1U}; /* p */ +uint64_t const CredentialsHolder::s_defaultBlockSize{8U}; /* r */ + +SensitiveData::SensitiveData(std::string&& data) : + d_data(std::move(data)) +{ +#ifdef HAVE_LIBSODIUM + sodium_mlock(d_data.data(), d_data.size()); +#endif +} + +SensitiveData& SensitiveData::operator=(SensitiveData&& rhs) +{ + d_data = std::move(rhs.d_data); + return *this; +} + +SensitiveData::SensitiveData(size_t bytes) +{ + d_data.resize(bytes); +#ifdef HAVE_LIBSODIUM + sodium_mlock(d_data.data(), d_data.size()); +#endif +} + +SensitiveData::~SensitiveData() +{ + clear(); +} + +void SensitiveData::clear() +{ +#ifdef HAVE_LIBSODIUM + sodium_munlock(d_data.data(), d_data.size()); +#endif + d_data.clear(); +} + +static std::string hashPasswordInternal(const std::string& password, const std::string& salt, uint64_t workFactor, uint64_t parallelFactor, uint64_t blockSize) +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + auto pctx = std::unique_ptr(EVP_PKEY_CTX_new_id(EVP_PKEY_SCRYPT, nullptr), EVP_PKEY_CTX_free); + if (!pctx) { + throw std::runtime_error("Error getting a scrypt context to hash the supplied password"); + } + + if (EVP_PKEY_derive_init(pctx.get()) <= 0) { + throw std::runtime_error("Error intializing the scrypt context to hash the supplied password"); + } + + // OpenSSL 3.0 changed the string arg to const unsigned char*, other versions use const char * +#if OPENSSL_VERSION_MAJOR >= 3 + auto passwordData = reinterpret_cast(password.data()); +#else + auto passwordData = reinterpret_cast(password.data()); +#endif + if (EVP_PKEY_CTX_set1_pbe_pass(pctx.get(), passwordData, password.size()) <= 0) { + throw std::runtime_error("Error adding the password to the scrypt context to hash the supplied password"); + } + + if (EVP_PKEY_CTX_set1_scrypt_salt(pctx.get(), reinterpret_cast(salt.data()), salt.size()) <= 0) { + throw std::runtime_error("Error adding the salt to the scrypt context to hash the supplied password"); + } + + if (EVP_PKEY_CTX_set_scrypt_N(pctx.get(), workFactor) <= 0) { + throw std::runtime_error("Error setting the work factor to the scrypt context to hash the supplied password"); + } + + if (EVP_PKEY_CTX_set_scrypt_r(pctx.get(), blockSize) <= 0) { + throw std::runtime_error("Error setting the block size to the scrypt context to hash the supplied password"); + } + + if (EVP_PKEY_CTX_set_scrypt_p(pctx.get(), parallelFactor) <= 0) { + throw std::runtime_error("Error setting the parallel factor to the scrypt context to hash the supplied password"); + } + + std::string out; + out.resize(pwhash_output_size); + size_t outlen = out.size(); + + if (EVP_PKEY_derive(pctx.get(), reinterpret_cast(out.data()), &outlen) <= 0 || outlen != pwhash_output_size) { + throw std::runtime_error("Error deriving the output from the scrypt context to hash the supplied password"); + } + + return out; +#else + throw std::runtime_error("Hashing support is not available"); +#endif +} + +static std::string generateRandomSalt() +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + /* generate a random salt */ + std::string salt; + salt.resize(pwhash_salt_size); + + if (RAND_bytes(reinterpret_cast(salt.data()), salt.size()) != 1) { + throw std::runtime_error("Error while generating a salt to hash the supplied password"); + } + + return salt; +#else + throw std::runtime_error("Generating a salted password requires scrypt support in OpenSSL, and it is not available"); +#endif +} + +std::string hashPassword(const std::string& password, uint64_t workFactor, uint64_t parallelFactor, uint64_t blockSize) +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + std::string result; + result.reserve(pwhash_max_size); + + result.append(pwhash_prefix); + result.append("ln="); + result.append(std::to_string(static_cast(std::log2(workFactor)))); + result.append(",p="); + result.append(std::to_string(parallelFactor)); + result.append(",r="); + result.append(std::to_string(blockSize)); + result.append("$"); + auto salt = generateRandomSalt(); + result.append(Base64Encode(salt)); + result.append("$"); + + auto out = hashPasswordInternal(password, salt, workFactor, parallelFactor, blockSize); + + result.append(Base64Encode(out)); + + return result; +#else + throw std::runtime_error("Hashing a password requires scrypt support in OpenSSL, and it is not available"); +#endif +} + +std::string hashPassword(const std::string& password) +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + return hashPassword(password, CredentialsHolder::s_defaultWorkFactor, CredentialsHolder::s_defaultParallelFactor, CredentialsHolder::s_defaultBlockSize); +#else + throw std::runtime_error("Hashing a password requires scrypt support in OpenSSL, and it is not available"); +#endif +} + +bool verifyPassword(const std::string& binaryHash, const std::string& salt, uint64_t workFactor, uint64_t parallelFactor, uint64_t blockSize, const std::string& binaryPassword) +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + auto expected = hashPasswordInternal(binaryPassword, salt, workFactor, parallelFactor, blockSize); + return constantTimeStringEquals(expected, binaryHash); +#else + throw std::runtime_error("Hashing a password requires scrypt support in OpenSSL, and it is not available"); +#endif +} + +/* parse a hashed password in PHC string format */ +static void parseHashed(const std::string& hash, std::string& salt, std::string& hashedPassword, uint64_t& workFactor, uint64_t& parallelFactor, uint64_t& blockSize) +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + auto parametersEnd = hash.find('$', pwhash_prefix.size()); + if (parametersEnd == std::string::npos || parametersEnd == hash.size()) { + throw std::runtime_error("Invalid hashed password format, no parameters"); + } + + auto parametersStr = hash.substr(pwhash_prefix.size(), parametersEnd); + std::vector parameters; + parameters.reserve(3); + stringtok(parameters, parametersStr, ","); + if (parameters.size() != 3) { + throw std::runtime_error("Invalid hashed password format, expecting 3 parameters, got " + std::to_string(parameters.size())); + } + + if (!boost::starts_with(parameters.at(0), "ln=")) { + throw std::runtime_error("Invalid hashed password format, ln= parameter not found"); + } + + if (!boost::starts_with(parameters.at(1), "p=")) { + throw std::runtime_error("Invalid hashed password format, p= parameter not found"); + } + + if (!boost::starts_with(parameters.at(2), "r=")) { + throw std::runtime_error("Invalid hashed password format, r= parameter not found"); + } + + auto saltPos = parametersEnd + 1; + auto saltEnd = hash.find('$', saltPos); + if (saltEnd == std::string::npos || saltEnd == hash.size()) { + throw std::runtime_error("Invalid hashed password format"); + } + + try { + workFactor = pdns_stou(parameters.at(0).substr(3)); + workFactor = static_cast(1) << workFactor; + if (workFactor > pwhash_max_work_factor) { + throw std::runtime_error("Invalid work factor of " + std::to_string(workFactor) + " in hashed password string, maximum is " + std::to_string(pwhash_max_work_factor)); + } + + parallelFactor = pdns_stou(parameters.at(1).substr(2)); + blockSize = pdns_stou(parameters.at(2).substr(2)); + + auto b64Salt = hash.substr(saltPos, saltEnd - saltPos); + salt.reserve(pwhash_salt_size); + B64Decode(b64Salt, salt); + + if (salt.size() != pwhash_salt_size) { + throw std::runtime_error("Invalid salt in hashed password string"); + } + + hashedPassword.reserve(pwhash_output_size); + B64Decode(hash.substr(saltEnd + 1), hashedPassword); + + if (hashedPassword.size() != pwhash_output_size) { + throw std::runtime_error("Invalid hash in hashed password string"); + } + } + catch (const std::exception& e) { + throw std::runtime_error("Invalid hashed password format, unable to parse parameters"); + } +#endif +} + +bool verifyPassword(const std::string& hash, const std::string& password) +{ + if (!isPasswordHashed(hash)) { + return false; + } + +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + std::string salt; + std::string hashedPassword; + uint64_t workFactor = 0; + uint64_t parallelFactor = 0; + uint64_t blockSize = 0; + parseHashed(hash, salt, hashedPassword, workFactor, parallelFactor, blockSize); + + auto expected = hashPasswordInternal(password, salt, workFactor, parallelFactor, blockSize); + + return constantTimeStringEquals(expected, hashedPassword); +#else + throw std::runtime_error("Verifying a hashed password requires scrypt support in OpenSSL, and it is not available"); +#endif +} + +bool isPasswordHashed(const std::string& password) +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + if (password.size() < pwhash_prefix_size || password.size() > pwhash_max_size) { + return false; + } + + if (!boost::starts_with(password, pwhash_prefix)) { + return false; + } + + auto parametersEnd = password.find('$', pwhash_prefix.size()); + if (parametersEnd == std::string::npos || parametersEnd == password.size()) { + return false; + } + + size_t parametersSize = parametersEnd - pwhash_prefix.size(); + /* ln=X,p=Y,r=Z */ + if (parametersSize < 12) { + return false; + } + + auto saltEnd = password.find('$', parametersEnd + 1); + if (saltEnd == std::string::npos || saltEnd == password.size()) { + return false; + } + + /* the salt is base64 encoded so it has to be larger than that */ + if ((saltEnd - parametersEnd - 1) < pwhash_salt_size) { + return false; + } + + /* the hash base64 encoded so it has to be larger than that */ + if ((password.size() - saltEnd - 1) < pwhash_output_size) { + return false; + } + + return true; +#else + return false; +#endif +} + +/* if the password is in cleartext and hashing is available, + the hashed form will be kept in memory */ +CredentialsHolder::CredentialsHolder(std::string&& password, bool hashPlaintext) : + d_credentials(std::move(password)) +{ + if (isHashingAvailable()) { + if (!isPasswordHashed(d_credentials.getString())) { + if (hashPlaintext) { + d_salt = generateRandomSalt(); + d_workFactor = s_defaultWorkFactor; + d_parallelFactor = s_defaultParallelFactor; + d_blockSize = s_defaultBlockSize; + d_credentials = SensitiveData(hashPasswordInternal(d_credentials.getString(), d_salt, d_workFactor, d_parallelFactor, d_blockSize)); + d_isHashed = true; + } + } + else { + d_wasHashed = true; + d_isHashed = true; + std::string hashedPassword; + parseHashed(d_credentials.getString(), d_salt, hashedPassword, d_workFactor, d_parallelFactor, d_blockSize); + d_credentials = SensitiveData(std::move(hashedPassword)); + } + } + + if (!d_isHashed) { + d_fallbackHashPerturb = random(); + d_fallbackHash = burtle(reinterpret_cast(d_credentials.getString().data()), d_credentials.getString().size(), d_fallbackHashPerturb); + } +} + +CredentialsHolder::~CredentialsHolder() +{ + d_fallbackHashPerturb = 0; + d_fallbackHash = 0; +} + +bool CredentialsHolder::matches(const std::string& password) const +{ + if (d_isHashed) { + return verifyPassword(d_credentials.getString(), d_salt, d_workFactor, d_parallelFactor, d_blockSize, password); + } + else { + uint32_t fallback = burtle(reinterpret_cast(password.data()), password.size(), d_fallbackHashPerturb); + if (fallback != d_fallbackHash) { + return false; + } + + return constantTimeStringEquals(password, d_credentials.getString()); + } +} + +bool CredentialsHolder::isHashingAvailable() +{ +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + return true; +#else + return false; +#endif +} + +#include +#include + +SensitiveData CredentialsHolder::readFromTerminal() +{ + struct termios term; + struct termios oterm; + bool restoreTermSettings = false; + int termAction = TCSAFLUSH; +#ifdef TCSASOFT + termAction |= TCSASOFT; +#endif + + FDWrapper input(open("/dev/tty", O_RDONLY)); + if (int(input) != -1) { + if (tcgetattr(input, &oterm) == 0) { + memcpy(&term, &oterm, sizeof(term)); + term.c_lflag &= ~(ECHO | ECHONL); + tcsetattr(input, termAction, &term); + restoreTermSettings = true; + } + } + else { + input = FDWrapper(dup(STDIN_FILENO)); + restoreTermSettings = false; + } + + FDWrapper output(open("/dev/tty", O_WRONLY)); + if (int(output) == -1) { + output = FDWrapper(dup(STDERR_FILENO)); + } + + struct std::map signals; + struct sigaction sa; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sa.sa_handler = [](int s) {}; + sigaction(SIGALRM, &sa, &signals[SIGALRM]); + sigaction(SIGHUP, &sa, &signals[SIGHUP]); + sigaction(SIGINT, &sa, &signals[SIGINT]); + sigaction(SIGPIPE, &sa, &signals[SIGPIPE]); + sigaction(SIGQUIT, &sa, &signals[SIGQUIT]); + sigaction(SIGTERM, &sa, &signals[SIGTERM]); + sigaction(SIGTSTP, &sa, &signals[SIGTSTP]); + sigaction(SIGTTIN, &sa, &signals[SIGTTIN]); + sigaction(SIGTTOU, &sa, &signals[SIGTTOU]); + + std::string buffer; + /* let's allocate a huge buffer now to prevent reallocation, + which would leave parts of the buffer around */ + buffer.reserve(512); + + for (;;) { + char ch = '\0'; + auto got = read(input, &ch, 1); + if (got == 1 && ch != '\n' && ch != '\r') { + buffer.push_back(ch); + } + else { + break; + } + } + + if (restoreTermSettings) { + tcsetattr(input, termAction, &oterm); + } + + for (const auto& sig : signals) { + sigaction(sig.first, &sig.second, nullptr); + } + + return SensitiveData(std::move(buffer)); +} diff --git a/pdns/credentials.hh b/pdns/credentials.hh new file mode 100644 index 000000000000..6e59c6b40e40 --- /dev/null +++ b/pdns/credentials.hh @@ -0,0 +1,103 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include + +class SensitiveData +{ +public: + SensitiveData(size_t bytes); + SensitiveData(std::string&& data); + SensitiveData& operator=(SensitiveData&&); + + ~SensitiveData(); + void clear(); + const std::string& getString() const + { + return d_data; + } + std::string& getString() + { + return d_data; + } + +private: + std::string d_data; +}; + +std::string hashPassword(const std::string& password); +std::string hashPassword(const std::string& password, uint64_t workFactor, uint64_t parallelFactor, uint64_t blockSize); +bool verifyPassword(const std::string& hash, const std::string& password); +bool verifyPassword(const std::string& binaryHash, const std::string& salt, uint64_t workFactor, uint64_t parallelFactor, uint64_t blockSize, const std::string& binaryPassword); +bool isPasswordHashed(const std::string& password); + +class CredentialsHolder +{ +public: + /* if hashPlaintext is true, the password is in cleartext and hashing is available, + the hashed form will be kept in memory. + Note that accepting hashed password from an untrusted source might open + us to a denial of service, since we currently don't cap the the parameters, + including the work factor */ + CredentialsHolder(std::string&& password, bool hashPlaintext); + ~CredentialsHolder(); + + CredentialsHolder(const CredentialsHolder&) = delete; + CredentialsHolder& operator=(const CredentialsHolder&) = delete; + + bool matches(const std::string& password) const; + /* whether it was constructed from a hashed and salted string */ + bool wasHashed() const + { + return d_wasHashed; + } + /* whether it is hashed in memory */ + bool isHashed() const + { + return d_isHashed; + } + + static bool isHashingAvailable(); + static SensitiveData readFromTerminal(); + + static uint64_t const s_defaultWorkFactor; + static uint64_t const s_defaultParallelFactor; + static uint64_t const s_defaultBlockSize; + +private: + SensitiveData d_credentials; + /* if the password is hashed, we only extract + the salt and parameters once */ + std::string d_salt; + uint64_t d_workFactor{0}; + uint64_t d_parallelFactor{0}; + uint64_t d_blockSize{0}; + /* seed our hash so it's not predictable */ + uint32_t d_fallbackHashPerturb{0}; + uint32_t d_fallbackHash{0}; + /* whether it was constructed from a hashed and salted string */ + bool d_wasHashed{false}; + /* whether it is hashed in memory */ + bool d_isHashed{false}; +}; diff --git a/pdns/dbdnsseckeeper.cc b/pdns/dbdnsseckeeper.cc index 650580a53ee6..0321ecc012ac 100644 --- a/pdns/dbdnsseckeeper.cc +++ b/pdns/dbdnsseckeeper.cc @@ -46,11 +46,9 @@ using namespace boost::assign; #include "namespaces.hh" -DNSSECKeeper::keycache_t DNSSECKeeper::s_keycache; -DNSSECKeeper::metacache_t DNSSECKeeper::s_metacache; +SharedLockGuarded DNSSECKeeper::s_keycache; +SharedLockGuarded DNSSECKeeper::s_metacache; int64_t DNSSECKeeper::s_metaCacheCleanActions = 0; -ReadWriteLock DNSSECKeeper::s_metacachelock; -ReadWriteLock DNSSECKeeper::s_keycachelock; AtomicCounter DNSSECKeeper::s_ops; time_t DNSSECKeeper::s_last_prune; size_t DNSSECKeeper::s_maxEntries = 0; @@ -119,34 +117,29 @@ bool DNSSECKeeper::addKey(const DNSName& name, bool setSEPBit, int algorithm, in } void DNSSECKeeper::clearAllCaches() { - { - WriteLock l(&s_keycachelock); - s_keycache.clear(); - } - WriteLock l(&s_metacachelock); - s_metacache.clear(); + s_keycache.write_lock()->clear(); + s_metacache.write_lock()->clear(); } - +/* This function never fails, the return value is to simplify call chains + elsewhere so we can do mutate && clear */ bool DNSSECKeeper::clearKeyCache(const DNSName& name) { - WriteLock l(&s_keycachelock); - s_keycache.erase(name); + s_keycache.write_lock()->erase(name); return true; } bool DNSSECKeeper::clearMetaCache(const DNSName& name) { - WriteLock l(&s_metacachelock); - s_metacache.erase(name); + s_metacache.write_lock()->erase(name); ++s_metaCacheCleanActions; return true; } void DNSSECKeeper::clearCaches(const DNSName& name) { - clearKeyCache(name); - clearMetaCache(name); + (void)clearKeyCache(name); + (void)clearMetaCache(name); } bool DNSSECKeeper::addKey(const DNSName& name, const DNSSECPrivateKey& dpk, int64_t& id, bool active, bool published) @@ -163,8 +156,8 @@ bool DNSSECKeeper::addKey(const DNSName& name, const DNSSECPrivateKey& dpk, int6 static bool keyCompareByKindAndID(const DNSSECKeeper::keyset_t::value_type& a, const DNSSECKeeper::keyset_t::value_type& b) { - return make_pair(!a.second.keyType, a.second.id) < - make_pair(!b.second.keyType, b.second.id); + return pair(!a.second.keyType, a.second.id) < + pair(!b.second.keyType, b.second.id); } DNSSECPrivateKey DNSSECKeeper::getKeyById(const DNSName& zname, unsigned int id) @@ -244,9 +237,9 @@ bool DNSSECKeeper::getFromMeta(const DNSName& zname, const std::string& key, std METAValues meta; if (ttl) { - ReadLock l(&s_metacachelock); - auto iter = s_metacache.find(zname); - if(iter != s_metacache.end() && iter->d_ttd > now) { + auto metacache = s_metacache.read_lock(); + auto iter = metacache->find(zname); + if(iter != metacache->end() && iter->d_ttd > now) { meta = iter->d_value; fromCache = true; } @@ -273,11 +266,11 @@ bool DNSSECKeeper::getFromMeta(const DNSName& zname, const std::string& key, std nce.d_ttd = now + ttl; nce.d_value = std::move(meta); { - WriteLock l(&s_metacachelock); + auto metacache = s_metacache.write_lock(); if(d_metaCacheCleanAction != s_metaCacheCleanActions) { return false; } - lruReplacingInsert(s_metacache, nce); + lruReplacingInsert(*metacache, nce); } } @@ -321,12 +314,10 @@ void DNSSECKeeper::getSoaEdit(const DNSName& zname, std::string& value, bool use uint64_t DNSSECKeeper::dbdnssecCacheSizes(const std::string& str) { if(str=="meta-cache-size") { - ReadLock l(&s_metacachelock); - return s_metacache.size(); + return s_metacache.read_lock()->size(); } else if(str=="key-cache-size") { - ReadLock l(&s_keycachelock); - return s_keycache.size(); + return s_keycache.read_lock()->size(); } return (uint64_t)-1; } @@ -553,10 +544,10 @@ DNSSECKeeper::keyset_t DNSSECKeeper::getKeys(const DNSName& zone, bool useCache) } if (useCache && ttl > 0) { - ReadLock l(&s_keycachelock); - keycache_t::const_iterator iter = s_keycache.find(zone); + auto keycache = s_keycache.read_lock(); + auto iter = keycache->find(zone); - if(iter != s_keycache.end() && iter->d_ttd > now) { + if (iter != keycache->end() && iter->d_ttd > now) { keyset_t ret; ret.reserve(iter->d_keys.size()); for(const keyset_t::value_type& value : iter->d_keys) @@ -613,7 +604,7 @@ DNSSECKeeper::keyset_t DNSSECKeeper::getKeys(const DNSName& zone, bool useCache) else kmd.keyType = ZSK; - retkeyset.push_back(make_pair(dpk, kmd)); + retkeyset.emplace_back(dpk, kmd); } sort(retkeyset.begin(), retkeyset.end(), keyCompareByKindAndID); @@ -623,8 +614,7 @@ DNSSECKeeper::keyset_t DNSSECKeeper::getKeys(const DNSName& zone, bool useCache) kce.d_keys = retkeyset; kce.d_ttd = now + ttl; { - WriteLock l(&s_keycachelock); - lruReplacingInsert(s_keycache, kce); + lruReplacingInsert(*(s_keycache.write_lock()), kce); } } @@ -751,7 +741,7 @@ bool DNSSECKeeper::rectifyZone(const DNSName& zone, string& error, string& info, } // We don't have a *full* Ueberbackend, just a key-only one. // Let's create one and use it - b = std::unique_ptr(new UeberBackend()); + b = std::make_unique(); B = b.get(); } @@ -978,12 +968,10 @@ void DNSSECKeeper::cleanup() if(now.tv_sec - s_last_prune > (time_t)(30)) { { - WriteLock l(&s_metacachelock); - pruneCollection(*this, s_metacache, s_maxEntries); + pruneCollection(*this, (*s_metacache.write_lock()), s_maxEntries); } { - WriteLock l(&s_keycachelock); - pruneCollection(*this, s_keycache, s_maxEntries); + pruneCollection(*this, (*s_keycache.write_lock()), s_maxEntries); } s_last_prune = time(nullptr); } @@ -993,7 +981,6 @@ void DNSSECKeeper::setMaxEntries(size_t maxEntries) { s_maxEntries = maxEntries; #if BOOST_VERSION >= 105600 - WriteLock wl(&s_keycachelock); - s_keycache.get().reserve(s_maxEntries); + s_keycache.write_lock()->get().reserve(s_maxEntries); #endif /* BOOST_VERSION >= 105600 */ } diff --git a/pdns/decafsigners.cc b/pdns/decafsigners.cc index 24df3d6ae2f3..cce6fd0479e3 100644 --- a/pdns/decafsigners.cc +++ b/pdns/decafsigners.cc @@ -67,10 +67,10 @@ DNSCryptoKeyEngine::storvector_t DecafED25519DNSCryptoKeyEngine::convertToISCVec PrivateKey: ODIyNjAzODQ2MjgwODAxMjI2NDUxOTAyMDQxNDIyNjI= */ - storvector_t storvector; - - storvector.push_back(make_pair("Algorithm", "15 (ED25519)")); - storvector.push_back(make_pair("PrivateKey", string((char*)d_seckey, DECAF_EDDSA_25519_PRIVATE_BYTES))); + auto storvector = storvector_t{ + {"Algorithm", "15 (ED25519)"}, + {"PrivateKey", string((char*)d_seckey, DECAF_EDDSA_25519_PRIVATE_BYTES)}, + }; return storvector; } @@ -203,10 +203,10 @@ DNSCryptoKeyEngine::storvector_t DecafED448DNSCryptoKeyEngine::convertToISCVecto PrivateKey: xZ+5Cgm463xugtkY5B0Jx6erFTXp13rYegst0qRtNsOYnaVpMx0Z/c5EiA9x8wWbDDct/U3FhYWA */ - storvector_t storvector; - - storvector.push_back(make_pair("Algorithm", "16 (ED448)")); - storvector.push_back(make_pair("PrivateKey", string((char*)d_seckey, DECAF_EDDSA_448_PRIVATE_BYTES))); + auto storvector = storvector_t{ + {"Algorithm", "16 (ED448)"}, + {"PrivateKey", string((char*)d_seckey, DECAF_EDDSA_448_PRIVATE_BYTES)}, + }; return storvector; } diff --git a/pdns/delaypipe.cc b/pdns/delaypipe.cc index 167f355fd7b8..be363d639191 100644 --- a/pdns/delaypipe.cc +++ b/pdns/delaypipe.cc @@ -162,7 +162,7 @@ void DelayPipe::worker() if(delay != 0 ) { int ret = d_pipe.readTimeout(&c, delay); if(ret > 0) { // we got an object - d_work.insert(make_pair(c.when, c.what)); + d_work.emplace(c.when, c.what); } else if(ret==0) { // EOF break; diff --git a/pdns/distributor.hh b/pdns/distributor.hh index 430f0981392d..a54b7099a97d 100644 --- a/pdns/distributor.hh +++ b/pdns/distributor.hh @@ -101,8 +101,8 @@ public: } Question Q; - callback_t callback; - int id; + callback_t callback{nullptr}; + int id{0}; }; bool isOverloaded() override @@ -160,7 +160,7 @@ templateMultiThreadDistributorMultiThreadDistributorvoid MultiThreadDistributor::distribute(int ournum) { + // this is the longest name we can use, not a typo setThreadName("pdns/distributo"); try { diff --git a/pdns/dns.cc b/pdns/dns.cc index 4bdd409d80b6..01a63d801315 100644 --- a/pdns/dns.cc +++ b/pdns/dns.cc @@ -79,64 +79,25 @@ std::string Opcode::to_s(uint8_t opcode) { return s_opcodes.at(opcode); } -class BoundsCheckingPointer -{ -public: - explicit BoundsCheckingPointer(const char* a, size_t length) - : d_ptr(a), d_length(length) - {} - - explicit BoundsCheckingPointer(const std::string& str) - : d_ptr(str.c_str()), d_length(str.size()) - {} - - - char operator[](size_t offset) const - { - if(offset < d_length) - return d_ptr[offset]; - throw runtime_error("out of bounds: "+std::to_string(offset)+" >= " + std::to_string(d_length)); - } -private: - const char* d_ptr; - const size_t d_length; -}; - - // goal is to hash based purely on the question name, and turn error into 'default' -uint32_t hashQuestion(const char* packet, uint16_t len, uint32_t init) +uint32_t hashQuestion(const uint8_t* packet, uint16_t packet_len, uint32_t init) { - if(len < 12) + if (packet_len < sizeof(dnsheader)) { return init; - - uint32_t ret=init; - const unsigned char* end = (const unsigned char*)packet+len; - const unsigned char* pos = (const unsigned char*)packet+12; + } + // C++ 17 does not have std::u8string_view + std::basic_string_view name(packet + sizeof(dnsheader), packet_len - sizeof(dnsheader)); + std::basic_string_view::size_type len = 0; - unsigned char labellen; - while((labellen=*pos++) && pos < end) { - if(pos + labellen + 1 > end) // include length field in hash - return 0; - ret=burtleCI(pos, labellen+1, ret); - pos += labellen; + while (len < name.length()) { + uint8_t labellen = name[len++]; + if (labellen == 0) { + // len is name.length() at max as it was < before the increment + return burtleCI(name.data(), len, init); + } + len += labellen; } - return ret; + // We've encountered a label that is too long + return init; } - -string& attodot(string &str) -{ - if(str.find_first_of('@')==string::npos) - return str; - - for (unsigned int i = 0; i < str.length(); i++) - { - if (str[i] == '@') { - str[i] = '.'; - break; - } else if (str[i] == '.') { - str.insert(i++, "\\"); - } - } - return str; -} diff --git a/pdns/dns.hh b/pdns/dns.hh index b5ca2ada1fd4..a23ad5ba1579 100644 --- a/pdns/dns.hh +++ b/pdns/dns.hh @@ -236,12 +236,10 @@ inline uint16_t * getFlagsFromDNSHeader(struct dnsheader * dh) extern time_t s_starttime; -uint32_t hashQuestion(const char* packet, uint16_t len, uint32_t init); +uint32_t hashQuestion(const uint8_t* packet, uint16_t len, uint32_t init); struct TSIGTriplet { DNSName name, algo; string secret; }; - -string &attodot(string &str); //!< for when you need to insert an email address in the SOA diff --git a/pdns/dns_random.hh b/pdns/dns_random.hh index b968b0766b0d..52f2812314af 100644 --- a/pdns/dns_random.hh +++ b/pdns/dns_random.hh @@ -48,4 +48,3 @@ namespace pdns { } }; } - diff --git a/pdns/dnsbackend.cc b/pdns/dnsbackend.cc index f2c86729c881..991ab6f3d119 100644 --- a/pdns/dnsbackend.cc +++ b/pdns/dnsbackend.cc @@ -171,7 +171,7 @@ void BackendMakerClass::launch(const string &instr) throw ArgException("Trying to launch unknown backend '"+module+"'"); } d_repository[module]->declareArguments(name); - d_instances.push_back(make_pair(module,name)); + d_instances.emplace_back(module, name); } } @@ -246,25 +246,31 @@ bool DNSBackend::getSOA(const DNSName &domain, SOAData &sd) S.inc("backend-queries"); DNSResourceRecord rr; - rr.auth = true; - int hits=0; - while(this->get(rr)) { - if (rr.qtype != QType::SOA) throw PDNSException("Got non-SOA record when asking for SOA"); - hits++; - fillSOAData(rr.content, sd); - sd.domain_id=rr.domain_id; - sd.ttl=rr.ttl; - } - - if(!hits) - return false; + sd.db = nullptr; - sd.qname = domain; - sd.db=this; + try { + while (this->get(rr)) { + if (rr.qtype != QType::SOA) { + throw PDNSException("Got non-SOA record when asking for SOA, zone: '" + domain.toLogString() + "'"); + } + hits++; + sd.qname = domain; + sd.ttl = rr.ttl; + sd.db = this; + sd.domain_id = rr.domain_id; + fillSOAData(rr.content, sd); + } + } + catch (...) { + while (this->get(rr)) { + ; + } + throw; + } - return true; + return hits; } bool DNSBackend::get(DNSZoneRecord& dzr) @@ -298,7 +304,7 @@ bool DNSBackend::getBeforeAndAfterNames(uint32_t id, const DNSName& zonename, co return ret; } -void DNSBackend::getAllDomains(vector* domains, bool include_disabled) +void DNSBackend::getAllDomains(vector* domains, bool getSerial, bool include_disabled) { if (g_zoneCache.isEnabled()) { g_log << Logger::Error << "One of the backends does not support zone caching. Put zone-cache-refresh-interval=0 in the config file to disable this cache." << endl; @@ -340,7 +346,7 @@ void fillSOAData(const string &content, SOAData &data) try { data.nameserver = DNSName(parts.at(0)); - data.hostmaster = DNSName(attodot(parts.at(1))); // ahu@ds9a.nl -> ahu.ds9a.nl, piet.puk@ds9a.nl -> piet\.puk.ds9a.nl + data.hostmaster = DNSName(parts.at(1)); data.serial = pdns_stou(parts.at(2).c_str()); data.refresh = pdns_stou(parts.at(3).c_str()); data.retry = pdns_stou(parts.at(4).c_str()); @@ -348,6 +354,6 @@ void fillSOAData(const string &content, SOAData &data) data.minimum = pdns_stou(parts.at(6).c_str()); } catch(const std::out_of_range& oor) { - throw PDNSException("Out of range exception parsing "+content); + throw PDNSException("Out of range exception parsing '" + content + "'"); } } diff --git a/pdns/dnsbackend.hh b/pdns/dnsbackend.hh index f768515fea08..c63033d7484e 100644 --- a/pdns/dnsbackend.hh +++ b/pdns/dnsbackend.hh @@ -173,7 +173,7 @@ public: return setDomainMetadata(name, kind, meta); } - virtual void getAllDomains(vector* domains, bool include_disabled = false); + virtual void getAllDomains(vector* domains, bool getSerial, bool include_disabled); /** Determines if we are authoritative for a zone, and at what level */ virtual bool getAuth(const DNSName &target, SOAData *sd); diff --git a/pdns/dnscrypt.cc b/pdns/dnscrypt.cc index 9a803958fdff..74d9182cfc6e 100644 --- a/pdns/dnscrypt.cc +++ b/pdns/dnscrypt.cc @@ -50,7 +50,7 @@ void DNSCryptPrivateKey::loadFromFile(const std::string& keyFile) void DNSCryptPrivateKey::saveToFile(const std::string& keyFile) const { ofstream file(keyFile); - file.write((char*) key, sizeof(key)); + file.write(reinterpret_cast(key), sizeof(key)); file.close(); } @@ -247,7 +247,7 @@ void DNSCryptContext::loadCertFromFile(const std::string&filename, DNSCryptCert& void DNSCryptContext::saveCertFromFile(const DNSCryptCert& cert, const std::string&filename) { ofstream file(filename); - file.write((char *) &cert, sizeof(cert)); + file.write(reinterpret_cast(&cert), sizeof(cert)); file.close(); } diff --git a/pdns/dnsdist-cache.cc b/pdns/dnsdist-cache.cc index 8b41f6845480..d023dee94239 100644 --- a/pdns/dnsdist-cache.cc +++ b/pdns/dnsdist-cache.cc @@ -26,7 +26,6 @@ #include "dnsparser.hh" #include "dnsdist-cache.hh" #include "dnsdist-ecs.hh" -#include "ednsoptions.hh" #include "ednssubnet.hh" #include "packetcache.hh" @@ -417,9 +416,9 @@ uint32_t DNSDistPacketCache::getKey(const DNSName::string_t& qname, size_t qname throw std::range_error("Computing packet cache key for an invalid packet (" + std::to_string(packet.size()) + " < " + std::to_string(sizeof(dnsheader) + qnameWireLength) + ")"); } if (packet.size() > ((sizeof(dnsheader) + qnameWireLength))) { - if (!d_cookieHashing) { - /* skip EDNS Cookie options if any */ - result = PacketCache::hashAfterQname(pdns_string_view(reinterpret_cast(packet.data()), packet.size()), result, sizeof(dnsheader) + qnameWireLength, false); + if (!d_optionsToSkip.empty()) { + /* skip EDNS options if any */ + result = PacketCache::hashAfterQname(pdns_string_view(reinterpret_cast(packet.data()), packet.size()), result, sizeof(dnsheader) + qnameWireLength, d_optionsToSkip); } else { result = burtle(&packet.at(sizeof(dnsheader) + qnameWireLength), packet.size() - (sizeof(dnsheader) + qnameWireLength), result); @@ -480,3 +479,8 @@ uint64_t DNSDistPacketCache::dump(int fd) return count; } + +void DNSDistPacketCache::setSkippedOptions(const std::unordered_set& optionsToSkip) +{ + d_optionsToSkip = optionsToSkip; +} diff --git a/pdns/dnsdist-cache.hh b/pdns/dnsdist-cache.hh index 0fbcb772b922..3309459ecb9b 100644 --- a/pdns/dnsdist-cache.hh +++ b/pdns/dnsdist-cache.hh @@ -28,6 +28,7 @@ #include "lock.hh" #include "noinitvector.hh" #include "stat_t.hh" +#include "ednsoptions.hh" struct DNSQuestion; @@ -54,9 +55,9 @@ public: uint64_t getTTLTooShorts() const { return d_ttlTooShorts; } uint64_t getEntriesCount(); uint64_t dump(int fd); + void setSkippedOptions(const std::unordered_set& optionsToSkip); bool isECSParsingEnabled() const { return d_parseECS; } - bool isCookieHashingEnabled() const { return d_cookieHashing; } bool keepStaleData() const { @@ -67,10 +68,6 @@ public: d_keepStaleData = keep; } - void setCookieHashing(bool hashing) - { - d_cookieHashing = hashing; - } void setECSParsingEnabled(bool enabled) { @@ -124,6 +121,7 @@ private: void insertLocked(CacheShard& shard, std::unordered_map& map, uint32_t key, CacheValue& newValue); std::vector d_shards; + std::unordered_set d_optionsToSkip{EDNSOptionCode::COOKIE}; pdns::stat_t d_deferredLookups{0}; pdns::stat_t d_deferredInserts{0}; @@ -144,5 +142,4 @@ private: bool d_deferrableInsertLock; bool d_parseECS; bool d_keepStaleData{false}; - bool d_cookieHashing{false}; }; diff --git a/pdns/dnsdist-console.cc b/pdns/dnsdist-console.cc index 4a5599519de4..44e33cc027dc 100644 --- a/pdns/dnsdist-console.cc +++ b/pdns/dnsdist-console.cc @@ -105,7 +105,7 @@ static void feedConfigDelta(const std::string& line) return; struct timeval now; gettimeofday(&now, 0); - g_confDelta.push_back({now,line}); + g_confDelta.emplace_back(now, line); } static string historyFile(const bool &ignoreHOME = false) @@ -403,6 +403,7 @@ const std::vector g_consoleKeywords{ { "addACL", true, "netmask", "add to the ACL set who can use this server" }, { "addAction", true, "DNS rule, DNS action [, {uuid=\"UUID\", name=\"name\"}]", "add a rule" }, { "addBPFFilterDynBlocks", true, "addresses, dynbpf[[, seconds=10], msg]", "This is the eBPF equivalent of addDynBlocks(), blocking a set of addresses for (optionally) a number of seconds, using an eBPF dynamic filter" }, + { "addCapabilitiesToRetain", true, "capability or list of capabilities", "Linux capabilities to retain after startup, like CAP_BPF" }, { "addConsoleACL", true, "netmask", "add a netmask to the console ACL" }, { "addDNSCryptBind", true, "\"127.0.0.1:8443\", \"provider name\", \"/path/to/resolver.cert\", \"/path/to/resolver.key\", {reusePort=false, tcpFastOpenQueueSize=0, interface=\"\", cpus={}}", "listen to incoming DNSCrypt queries on 127.0.0.1 port 8443, with a provider name of `provider name`, using a resolver certificate and associated key stored respectively in the `resolver.cert` and `resolver.key` files. The fifth optional parameter is a table of parameters" }, { "addDOHLocal", true, "addr, certFile, keyFile [, urls [, vars]]", "listen to incoming DNS over HTTPS queries on the specified address using the specified certificate and key. The last two parameters are tables" }, @@ -472,6 +473,7 @@ const std::vector g_consoleKeywords{ { "getTLSFrontend", true, "n", "returns the TLS frontend with index n" }, { "getTLSFrontendCount", true, "", "returns the number of DoT listeners" }, { "grepq", true, "Netmask|DNS Name|100ms|{\"::1\", \"powerdns.com\", \"100ms\"} [, n]", "shows the last n queries and responses matching the specified client address or range (Netmask), or the specified DNS Name, or slower than 100ms" }, + { "hashPassword", true, "password [, workFactor]", "Returns a hashed and salted version of the supplied password, usable with 'setWebserverConfig()'"}, { "HTTPHeaderRule", true, "name, regex", "matches DoH queries with a HTTP header 'name' whose content matches the regular expression 'regex'"}, { "HTTPPathRegexRule", true, "regex", "matches DoH queries whose HTTP path matches 'regex'"}, { "HTTPPathRule", true, "path", "matches DoH queries whose HTTP path is an exact match to 'path'"}, @@ -532,6 +534,7 @@ const std::vector g_consoleKeywords{ { "newServer", true, "{address=\"ip:port\", qps=1000, order=1, weight=10, pool=\"abuse\", retries=5, tcpConnectTimeout=5, tcpSendTimeout=30, tcpRecvTimeout=30, checkName=\"a.root-servers.net.\", checkType=\"A\", maxCheckFailures=1, mustResolve=false, useClientSubnet=true, source=\"address|interface name|address@interface\", sockets=1, reconnectOnUp=false}", "instantiate a server" }, { "newServerPolicy", true, "name, function", "create a policy object from a Lua function" }, { "newSuffixMatchNode", true, "", "returns a new SuffixMatchNode" }, + { "newSVCRecordParameters", true, "priority, target, mandatoryParams, alpns, noDefaultAlpn [, port [, ech [, ipv4hints [, ipv6hints [, additionalParameters ]]]]]", "return a new SVCRecordParameters object, to use with SpoofSVCAction" }, { "NegativeAndSOAAction", true, "nxd, zone, ttl, mname, rname, serial, refresh, retry, expire, minimum [, options]", "Turn a query into a NXDomain or NoData answer and sets a SOA record in the additional section" }, { "NoneAction", true, "", "Does nothing. Subsequent rules are processed after this action" }, { "NotRule", true, "selector", "Matches the traffic if the selector rule does not match" }, @@ -539,6 +542,7 @@ const std::vector g_consoleKeywords{ { "OrRule", true, "selectors", "Matches the traffic if one or more of the the selectors rules does match" }, { "PoolAction", true, "poolname", "set the packet into the specified pool" }, { "PoolAvailableRule", true, "poolname", "Check whether a pool has any servers available to handle queries" }, + { "PoolOutstandingRule", true, "poolname, limit", "Check whether a pool has outstanding queries above limit" }, { "printDNSCryptProviderFingerprint", true, "\"/path/to/providerPublic.key\"", "display the fingerprint of the provided resolver public key" }, { "ProbaRule", true, "probability", "Matches queries with a given probability. 1.0 means always" }, { "ProxyProtocolValueRule", true, "type [, value]", "matches queries with a specified Proxy Protocol TLV value of that type, optionally matching the content of the option as well" }, @@ -582,6 +586,8 @@ const std::vector g_consoleKeywords{ { "setConsoleMaximumConcurrentConnections", true, "max", "Set the maximum number of concurrent console connections" }, { "setConsoleOutputMaxMsgSize", true, "messageSize", "set console message maximum size in bytes, default is 10 MB" }, { "setDefaultBPFFilter", true, "filter", "When used at configuration time, the corresponding BPFFilter will be attached to every bind" }, + { "setDoHDownstreamCleanupInterval", true, "interval", "minimum interval in seconds between two cleanups of the idle DoH downstream connections" }, + { "setDoHDownstreamMaxIdleTime", true, "time", "Maximum time in seconds that a downstream DoH connection to a backend might stay idle" }, { "setDynBlocksAction", true, "action", "set which action is performed when a query is blocked. Only DNSAction.Drop (the default) and DNSAction.Refused are supported" }, { "setDynBlocksPurgeInterval", true, "sec", "set how often the expired dynamic block entries should be removed" }, { "setDropEmptyQueries", true, "drop", "Whether to drop empty queries right away instead of sending a NOTIMP response" }, @@ -590,6 +596,7 @@ const std::vector g_consoleKeywords{ { "setECSSourcePrefixV6", true, "prefix-length", "the EDNS Client Subnet prefix-length used for IPv6 queries" }, { "setKey", true, "key", "set access key to that key" }, { "setLocal", true, "addr [, {doTCP=true, reusePort=false, tcpFastOpenQueueSize=0, interface=\"\", cpus={}}]", "reset the list of addresses we listen on to this address" }, + { "setMaxCachedDoHConnectionsPerDownstream", true, "max", "Set the maximum number of inactive DoH connections to a backend cached by each worker DoH thread" }, { "setMaxCachedTCPConnectionsPerDownstream", true, "max", "Set the maximum number of inactive TCP connections to a backend cached by each worker TCP thread" }, { "setMaxTCPClientThreads", true, "n", "set the maximum of TCP client threads, handling TCP connections" }, { "setMaxTCPConnectionDuration", true, "n", "set the maximum duration of an incoming TCP connection, in seconds. 0 means unlimited" }, @@ -621,6 +628,7 @@ const std::vector g_consoleKeywords{ { "setStaleCacheEntriesTTL", true, "n", "allows using cache entries expired for at most n seconds when there is no backend available to answer for a query" }, { "setSyslogFacility", true, "facility", "set the syslog logging facility to 'facility'. Defaults to LOG_DAEMON" }, { "setTCPDownstreamCleanupInterval", true, "interval", "minimum interval in seconds between two cleanups of the idle TCP downstream connections" }, + { "setTCPDownstreamMaxIdleTime", true, "time", "Maximum time in seconds that a downstream TCP connection to a backend might stay idle" }, { "setTCPInternalPipeBufferSize", true, "size", "Set the size in bytes of the internal buffer of the pipes used internally to distribute connections to TCP (and DoT) workers threads" }, { "setTCPRecvTimeout", true, "n", "set the read timeout on TCP connections from the client, in seconds" }, { "setTCPSendTimeout", true, "n", "set the write timeout on TCP connections from the client, in seconds" }, @@ -661,7 +669,9 @@ const std::vector g_consoleKeywords{ { "SetECSOverrideAction", true, "override", "Whether an existing EDNS Client Subnet value should be overridden (true) or not (false). Subsequent rules are processed after this action" }, { "SetECSPrefixLengthAction", true, "v4, v6", "Set the ECS prefix length. Subsequent rules are processed after this action" }, { "SetMacAddrAction", true, "option", "Add the source MAC address to the query as EDNS0 option option. This action is currently only supported on Linux. Subsequent rules are processed after this action" }, + { "SetEDNSOptionAction", true, "option, data", "Add arbitrary EDNS option and data to the query. Subsequent rules are processed after this action" }, { "SetNoRecurseAction", true, "", "strip RD bit from the question, let it go through" }, + { "setOutgoingDoHWorkerThreads", true, "n", "Number of outgoing DoH worker threads" }, { "SetProxyProtocolValuesAction", true, "values", "Set the Proxy-Protocol values for this queries to 'values'" }, { "SetSkipCacheAction", true, "", "Don’t lookup the cache for this query, don’t store the answer" }, { "SetSkipCacheResponseAction", true, "", "Don’t store this response into the cache" }, @@ -674,6 +684,7 @@ const std::vector g_consoleKeywords{ { "SpoofAction", true, "ip|list of ips [, options]", "forge a response with the specified IPv4 (for an A query) or IPv6 (for an AAAA). If you specify multiple addresses, all that match the query type (A, AAAA or ANY) will get spoofed in" }, { "SpoofCNAMEAction", true, "cname [, options]", "Forge a response with the specified CNAME value" }, { "SpoofRawAction", true, "raw|list of raws [, options]", "Forge a response with the specified record data as raw bytes. If you specify multiple raws (it is assumed they match the query type), all will get spoofed in" }, + { "SpoofSVCAction", true, "list of svcParams [, options]", "Forge a response with the specified SVC record data" } , { "SuffixMatchNodeRule", true, "smn[, quiet]", "Matches based on a group of domain suffixes for rapid testing of membership. Pass true as second parameter to prevent listing of all domains matched" }, { "TagRule", true, "name [, value]", "matches if the tag named 'name' is present, with the given 'value' matching if any" }, { "TCAction", true, "", "create answer to query with TC and RD bits set, to move to TCP" }, diff --git a/pdns/dnsdist-dnscrypt.cc b/pdns/dnsdist-dnscrypt.cc index fe18169db98b..99301445c3f7 100644 --- a/pdns/dnsdist-dnscrypt.cc +++ b/pdns/dnsdist-dnscrypt.cc @@ -24,17 +24,17 @@ #include "dnscrypt.hh" #ifdef HAVE_DNSCRYPT -int handleDNSCryptQuery(PacketBuffer& packet, std::shared_ptr& query, bool tcp, time_t now, PacketBuffer& response) +int handleDNSCryptQuery(PacketBuffer& packet, DNSCryptQuery& query, bool tcp, time_t now, PacketBuffer& response) { - query->parsePacket(packet, tcp, now); + query.parsePacket(packet, tcp, now); - if (query->isValid() == false) { + if (query.isValid() == false) { vinfolog("Dropping DNSCrypt invalid query"); return false; } - if (query->isEncrypted() == false) { - query->getCertificateResponse(now, response); + if (query.isEncrypted() == false) { + query.getCertificateResponse(now, response); return false; } diff --git a/pdns/dnsdist-dynblocks.hh b/pdns/dnsdist-dynblocks.hh index d802d05e1e1c..2a817e13bfe7 100644 --- a/pdns/dnsdist-dynblocks.hh +++ b/pdns/dnsdist-dynblocks.hh @@ -45,13 +45,14 @@ typedef std::function dnsdist_ffi_stat_node_visi struct dnsdist_ffi_stat_node_t { - dnsdist_ffi_stat_node_t(const StatNode& node_, const StatNode::Stat& self_, const StatNode::Stat& children_): node(node_), self(self_), children(children_) + dnsdist_ffi_stat_node_t(const StatNode& node_, const StatNode::Stat& self_, const StatNode::Stat& children_, std::optional& reason_): node(node_), self(self_), children(children_), reason(reason_) { } const StatNode& node; const StatNode::Stat& self; const StatNode::Stat& children; + std::optional& reason; }; class DynBlockRulesGroup @@ -211,7 +212,7 @@ private: double d_warningRatio{0.0}; }; - typedef std::unordered_map counts_t; + typedef std::unordered_map counts_t; public: DynBlockRulesGroup() @@ -247,7 +248,7 @@ public: entry = DynBlockRule(reason, blockDuration, rate, warningRate, seconds, action); } - typedef std::function smtVisitor_t; + typedef std::function>(const StatNode&, const StatNode::Stat&, const StatNode::Stat&)> smtVisitor_t; void setSuffixMatchRule(unsigned int seconds, std::string reason, unsigned int blockDuration, DNSAction::Action action, smtVisitor_t visitor) { @@ -261,6 +262,13 @@ public: d_smtVisitorFFI = visitor; } + void setMasks(uint8_t v4, uint8_t v6, uint8_t port) + { + d_v4Mask = v4; + d_v6Mask = v6; + d_portMask = port; + } + void apply() { struct timespec now; @@ -329,15 +337,15 @@ private: bool checkIfQueryTypeMatches(const Rings::Query& query); bool checkIfResponseCodeMatches(const Rings::Response& response); - void addOrRefreshBlock(boost::optional >& blocks, const struct timespec& now, const ComboAddress& requestor, const DynBlockRule& rule, bool& updated, bool warning); + void addOrRefreshBlock(boost::optional >& blocks, const struct timespec& now, const AddressAndPortRange& requestor, const DynBlockRule& rule, bool& updated, bool warning); void addOrRefreshBlockSMT(SuffixMatchTree& blocks, const struct timespec& now, const DNSName& name, const DynBlockRule& rule, bool& updated); - void addBlock(boost::optional >& blocks, const struct timespec& now, const ComboAddress& requestor, const DynBlockRule& rule, bool& updated) + void addBlock(boost::optional >& blocks, const struct timespec& now, const AddressAndPortRange& requestor, const DynBlockRule& rule, bool& updated) { addOrRefreshBlock(blocks, now, requestor, rule, updated, false); } - void handleWarning(boost::optional >& blocks, const struct timespec& now, const ComboAddress& requestor, const DynBlockRule& rule, bool& updated) + void handleWarning(boost::optional >& blocks, const struct timespec& now, const AddressAndPortRange& requestor, const DynBlockRule& rule, bool& updated) { addOrRefreshBlock(blocks, now, requestor, rule, updated, true); } @@ -375,6 +383,9 @@ private: SuffixMatchNode d_excludedDomains; smtVisitor_t d_smtVisitor; dnsdist_ffi_stat_node_visitor_t d_smtVisitorFFI; + uint8_t d_v6Mask{128}; + uint8_t d_v4Mask{32}; + uint8_t d_portMask{0}; bool d_beQuiet{false}; }; @@ -384,11 +395,11 @@ public: static void run(); /* return the (cached) number of hits per second for the top offenders, averaged over 60s */ - static std::map>> getHitsForTopNetmasks(); + static std::map>> getHitsForTopNetmasks(); static std::map>> getHitsForTopSuffixes(); /* get the the top offenders based on the current value of the counters */ - static std::map>> getTopNetmasks(size_t topN); + static std::map>> getTopNetmasks(size_t topN); static std::map>> getTopSuffixes(size_t topN); static void purgeExpired(const struct timespec& now); @@ -400,13 +411,13 @@ private: struct MetricsSnapshot { - std::map>> nmgData; + std::map>> nmgData; std::map>> smtData; }; struct Tops { - std::map>> topNMGsByReason; + std::map>> topNMGsByReason; std::map>> topSMTsByReason; }; diff --git a/pdns/dnsdist-dynbpf.cc b/pdns/dnsdist-dynbpf.cc index a11c928f95cd..ec549c32f96b 100644 --- a/pdns/dnsdist-dynbpf.cc +++ b/pdns/dnsdist-dynbpf.cc @@ -38,7 +38,7 @@ bool DynBPFFilter::block(const ComboAddress& addr, const struct timespec& until) } } else { - data->d_bpf->block(addr); + data->d_bpf->block(addr, BPFFilter::MatchAction::Drop); data->d_entries.insert(BlockEntry(addr, until)); inserted = true; } diff --git a/pdns/dnsdist-ecs.cc b/pdns/dnsdist-ecs.cc index e7b6fdd0ccc3..e4e3bd6e8bd5 100644 --- a/pdns/dnsdist-ecs.cc +++ b/pdns/dnsdist-ecs.cc @@ -128,11 +128,11 @@ int rewriteResponseWithoutEDNS(const PacketBuffer& initialPacket, PacketBuffer& return 0; } -static bool addOrReplaceECSOption(std::vector>& options, bool& ecsAdded, bool overrideExisting, const string& newECSOption) +static bool addOrReplaceEDNSOption(std::vector>& options, uint16_t optionCode, bool& optionAdded, bool overrideExisting, const string& newOptionContent) { for (auto it = options.begin(); it != options.end(); ) { - if (it->first == EDNSOptionCode::ECS) { - ecsAdded = false; + if (it->first == optionCode) { + optionAdded = false; if (!overrideExisting) { return false; @@ -145,26 +145,26 @@ static bool addOrReplaceECSOption(std::vector>& } } - options.emplace_back(EDNSOptionCode::ECS, std::string(&newECSOption.at(EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE), newECSOption.size() - (EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE))); + options.emplace_back(optionCode, std::string(&newOptionContent.at(EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE), newOptionContent.size() - (EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE))); return true; } -static bool slowRewriteQueryWithRecords(const PacketBuffer& initialPacket, PacketBuffer& newContent, bool& ednsAdded, bool& ecsAdded, bool overrideExisting, const string& newECSOption) +bool slowRewriteEDNSOptionInQueryWithRecords(const PacketBuffer& initialPacket, PacketBuffer& newContent, bool& ednsAdded, uint16_t optionToReplace, bool& optionAdded, bool overrideExisting, const string& newOptionContent) { assert(initialPacket.size() >= sizeof(dnsheader)); const struct dnsheader* dh = reinterpret_cast(initialPacket.data()); - ecsAdded = false; - ednsAdded = true; - if (ntohs(dh->qdcount) == 0) { return false; } if (ntohs(dh->ancount) == 0 && ntohs(dh->nscount) == 0 && ntohs(dh->arcount) == 0) { - throw std::runtime_error("slowRewriteQueryWithRecords() should not be called for queries that have no records"); + throw std::runtime_error(std::string(__PRETTY_FUNCTION__) + " should not be called for queries that have no records"); } + optionAdded = false; + ednsAdded = true; + PacketReader pr(pdns_string_view(reinterpret_cast(initialPacket.data()), initialPacket.size())); size_t idx = 0; @@ -244,16 +244,16 @@ static bool slowRewriteQueryWithRecords(const PacketBuffer& initialPacket, Packe static_assert(sizeof(edns0) == sizeof(ah.d_ttl), "sizeof(EDNS0Record) must match sizeof(uint32_t) AKA RR TTL size"); memcpy(&edns0, &ah.d_ttl, sizeof(edns0)); - /* addOrReplaceECSOption will set it to false if there is already an existing option */ - ecsAdded = true; - addOrReplaceECSOption(options, ecsAdded, overrideExisting, newECSOption); + /* addOrReplaceEDNSOption will set it to false if there is already an existing option */ + optionAdded = true; + addOrReplaceEDNSOption(options, optionToReplace, optionAdded, overrideExisting, newOptionContent); pw.addOpt(ah.d_class, edns0.extRCode, edns0.extFlags, options, edns0.version); } } if (ednsAdded) { - pw.addOpt(g_EdnsUDPPayloadSize, 0, 0, {{EDNSOptionCode::ECS, std::string(&newECSOption.at(EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE), newECSOption.size() - (EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE))}}, 0); - ecsAdded = true; + pw.addOpt(g_EdnsUDPPayloadSize, 0, 0, {{optionToReplace, std::string(&newOptionContent.at(EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE), newOptionContent.size() - (EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE))}}, 0); + optionAdded = true; } pw.commit(); @@ -588,9 +588,7 @@ bool handleEDNSClientSubnet(PacketBuffer& packet, const size_t maximumSize, cons PacketBuffer newContent; newContent.reserve(packet.size()); - if (!slowRewriteQueryWithRecords(packet, newContent, ednsAdded, ecsAdded, overrideExisting, newECSOption)) { - ednsAdded = false; - ecsAdded = false; + if (!slowRewriteEDNSOptionInQueryWithRecords(packet, newContent, ednsAdded, EDNSOptionCode::ECS, ecsAdded, overrideExisting, newECSOption)) { return false; } diff --git a/pdns/dnsdist-ecs.hh b/pdns/dnsdist-ecs.hh index 26f758b6dbbd..89e6237963f8 100644 --- a/pdns/dnsdist-ecs.hh +++ b/pdns/dnsdist-ecs.hh @@ -28,6 +28,7 @@ extern size_t g_EdnsUDPPayloadSize; extern uint16_t g_PayloadSizeSelfGenAnswers; int rewriteResponseWithoutEDNS(const PacketBuffer& initialPacket, PacketBuffer& newContent); +bool slowRewriteEDNSOptionInQueryWithRecords(const PacketBuffer& initialPacket, PacketBuffer& newContent, bool& ednsAdded, uint16_t optionToReplace, bool& optionAdded, bool overrideExisting, const string& newOptionContent); int locateEDNSOptRR(const PacketBuffer & packet, uint16_t * optStart, size_t * optLen, bool * last); bool generateOptRR(const std::string& optRData, PacketBuffer& res, size_t maximumSize, uint16_t udpPayloadSize, uint8_t ednsrcode, bool dnssecOK); void generateECSOption(const ComboAddress& source, string& res, uint16_t ECSPrefixLength); diff --git a/pdns/dnsdist-idstate.hh b/pdns/dnsdist-idstate.hh index 5d991042fda3..043a56911f72 100644 --- a/pdns/dnsdist-idstate.hh +++ b/pdns/dnsdist-idstate.hh @@ -44,9 +44,7 @@ struct StopWatch void start() { - if (gettime(&d_start, d_needRealTime) < 0) { - unixDie("Getting timestamp"); - } + d_start = getCurrentTime(); } void set(const struct timespec& from) @@ -56,24 +54,25 @@ struct StopWatch double udiff() const { - struct timespec now; - if (gettime(&now, d_needRealTime) < 0) { - unixDie("Getting timestamp"); - } - + struct timespec now = getCurrentTime(); return 1000000.0 * (now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec) / 1000.0; } double udiffAndSet() + { + struct timespec now = getCurrentTime(); + auto ret = 1000000.0 * (now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec) / 1000.0; + d_start = now; + return ret; + } + + struct timespec getCurrentTime() const { struct timespec now; if (gettime(&now, d_needRealTime) < 0) { unixDie("Getting timestamp"); } - - auto ret = 1000000.0 * (now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec) / 1000.0; - d_start = now; - return ret; + return now; } struct timespec d_start @@ -82,7 +81,7 @@ struct StopWatch }; private: - bool d_needRealTime{false}; + bool d_needRealTime; }; /* g++ defines __SANITIZE_THREAD__ @@ -100,7 +99,7 @@ struct IDState sentTime(true), tempFailureTTL(boost::none) { origDest.sin4.sin_family = 0; } IDState(const IDState& orig) = delete; IDState(IDState&& rhs) : - subnet(rhs.subnet), origRemote(rhs.origRemote), origDest(rhs.origDest), hopRemote(rhs.hopRemote), hopLocal(rhs.hopLocal), qname(std::move(rhs.qname)), sentTime(rhs.sentTime), dnsCryptQuery(std::move(rhs.dnsCryptQuery)), packetCache(std::move(rhs.packetCache)), qTag(std::move(rhs.qTag)), tempFailureTTL(rhs.tempFailureTTL), cs(rhs.cs), du(std::move(rhs.du)), cacheKey(rhs.cacheKey), cacheKeyNoECS(rhs.cacheKeyNoECS), cacheKeyUDP(rhs.cacheKeyUDP), origFD(rhs.origFD), delayMsec(rhs.delayMsec), qtype(rhs.qtype), qclass(rhs.qclass), origID(rhs.origID), origFlags(rhs.origFlags), cacheFlags(rhs.cacheFlags), protocol(rhs.protocol), ednsAdded(rhs.ednsAdded), ecsAdded(rhs.ecsAdded), skipCache(rhs.skipCache), destHarvested(rhs.destHarvested), dnssecOK(rhs.dnssecOK), useZeroScope(rhs.useZeroScope) + subnet(rhs.subnet), origRemote(rhs.origRemote), origDest(rhs.origDest), hopRemote(rhs.hopRemote), hopLocal(rhs.hopLocal), qname(std::move(rhs.qname)), sentTime(rhs.sentTime), packetCache(std::move(rhs.packetCache)), dnsCryptQuery(std::move(rhs.dnsCryptQuery)), qTag(std::move(rhs.qTag)), tempFailureTTL(rhs.tempFailureTTL), cs(rhs.cs), du(std::move(rhs.du)), cacheKey(rhs.cacheKey), cacheKeyNoECS(rhs.cacheKeyNoECS), cacheKeyUDP(rhs.cacheKeyUDP), origFD(rhs.origFD), delayMsec(rhs.delayMsec), qtype(rhs.qtype), qclass(rhs.qclass), origID(rhs.origID), origFlags(rhs.origFlags), cacheFlags(rhs.cacheFlags), protocol(rhs.protocol), ednsAdded(rhs.ednsAdded), ecsAdded(rhs.ecsAdded), skipCache(rhs.skipCache), destHarvested(rhs.destHarvested), dnssecOK(rhs.dnssecOK), useZeroScope(rhs.useZeroScope) { if (rhs.isInUse()) { throw std::runtime_error("Trying to move an in-use IDState"); @@ -237,9 +236,9 @@ struct IDState ComboAddress hopLocal; DNSName qname; // 24 StopWatch sentTime; // 16 - std::shared_ptr dnsCryptQuery{nullptr}; // 16 std::shared_ptr packetCache{nullptr}; // 16 - std::shared_ptr qTag{nullptr}; // 16 + std::unique_ptr dnsCryptQuery{nullptr}; // 8 + std::unique_ptr qTag{nullptr}; // 8 boost::optional tempFailureTTL; // 8 const ClientState* cs{nullptr}; // 8 DOHUnit* du{nullptr}; // 8 diff --git a/pdns/dnsdist-lua-actions.cc b/pdns/dnsdist-lua-actions.cc index 12df54ae93c1..03475f79b95e 100644 --- a/pdns/dnsdist-lua-actions.cc +++ b/pdns/dnsdist-lua-actions.cc @@ -27,12 +27,15 @@ #include "dnsdist-lua-ffi.hh" #include "dnsdist-protobuf.hh" #include "dnsdist-kvs.hh" +#include "dnsdist-svc.hh" #include "dolog.hh" #include "dnstap.hh" +#include "dnswriter.hh" #include "ednsoptions.hh" #include "fstrm_logger.hh" #include "remote_logger.hh" +#include "svc-records.hh" #include @@ -355,6 +358,97 @@ class ERCodeAction : public DNSAction uint8_t d_rcode; }; +class SpoofSVCAction : public DNSAction +{ +public: + SpoofSVCAction(const std::vector>& parameters) + { + d_payloads.reserve(parameters.size()); + + for (const auto& param : parameters) { + std::vector payload; + if (!generateSVCPayload(payload, param.second)) { + throw std::runtime_error("Unable to generate a valid SVC record from the supplied parameters"); + } + + d_totalPayloadsSize += payload.size(); + d_payloads.push_back(std::move(payload)); + + for (const auto& hint : param.second.ipv4hints) { + d_additionals4.insert({ param.second.target, ComboAddress(hint) }); + } + + for (const auto& hint : param.second.ipv6hints) { + d_additionals6.insert({ param.second.target, ComboAddress(hint) }); + } + } + } + + DNSAction::Action operator()(DNSQuestion* dq, std::string* ruleresult) const override + { + /* it will likely be a bit bigger than that because of additionals */ + uint16_t numberOfRecords = d_payloads.size(); + const auto qnameWireLength = dq->qname->wirelength(); + if (dq->getMaximumSize() < (sizeof(dnsheader) + qnameWireLength + 4 + numberOfRecords*12 /* recordstart */ + d_totalPayloadsSize)) { + return Action::None; + } + + PacketBuffer newPacket; + newPacket.reserve(sizeof(dnsheader) + qnameWireLength + 4 + numberOfRecords*12 /* recordstart */ + d_totalPayloadsSize); + GenericDNSPacketWriter pw(newPacket, *dq->qname, dq->qtype); + for (const auto& payload : d_payloads) { + pw.startRecord(*dq->qname, dq->qtype, d_responseConfig.ttl); + pw.xfrBlob(payload); + pw.commit(); + } + + if (newPacket.size() < dq->getMaximumSize()) { + for (const auto& additional : d_additionals4) { + pw.startRecord(additional.first.isRoot() ? *dq->qname : additional.first, QType::A, d_responseConfig.ttl, QClass::IN, DNSResourceRecord::ADDITIONAL); + pw.xfrCAWithoutPort(4, additional.second); + pw.commit(); + } + } + + if (newPacket.size() < dq->getMaximumSize()) { + for (const auto& additional : d_additionals6) { + pw.startRecord(additional.first.isRoot() ? *dq->qname : additional.first, QType::AAAA, d_responseConfig.ttl, QClass::IN, DNSResourceRecord::ADDITIONAL); + pw.xfrCAWithoutPort(6, additional.second); + pw.commit(); + } + } + + if (g_addEDNSToSelfGeneratedResponses && queryHasEDNS(*dq)) { + bool dnssecOK = getEDNSZ(*dq) & EDNS_HEADER_FLAG_DO; + pw.addOpt(g_PayloadSizeSelfGenAnswers, 0, dnssecOK ? EDNS_HEADER_FLAG_DO : 0); + pw.commit(); + } + + if (newPacket.size() >= dq->getMaximumSize()) { + /* sorry! */ + return Action::None; + } + + pw.getHeader()->id = dq->getHeader()->id; + pw.getHeader()->qr = true; // for good measure + setResponseHeadersFromConfig(*pw.getHeader(), d_responseConfig); + dq->getMutableData() = std::move(newPacket); + + return Action::HeaderModify; + } + std::string toString() const override + { + return "spoof SVC record "; + } + + ResponseConfig d_responseConfig; +private: + std::vector> d_payloads; + std::set> d_additionals4; + std::set> d_additionals6; + size_t d_totalPayloadsSize{0}; +}; + class TCAction : public DNSAction { public: @@ -722,7 +816,7 @@ DNSAction::Action SpoofAction::operator()(DNSQuestion* dq, std::string* ruleresu } unsigned int qnameWireLength=0; - DNSName ignore((char*)dq->getData().data(), dq->getData().size(), sizeof(dnsheader), false, 0, 0, &qnameWireLength); + DNSName ignore(reinterpret_cast(dq->getData().data()), dq->getData().size(), sizeof(dnsheader), false, 0, 0, &qnameWireLength); if (dq->getMaximumSize() < (sizeof(dnsheader) + qnameWireLength + 4 + numberOfRecords*12 /* recordstart */ + totrdatalen)) { return Action::None; @@ -794,7 +888,7 @@ DNSAction::Action SpoofAction::operator()(DNSQuestion* dq, std::string* ruleresu dest += sizeof(recordstart); memcpy(dest, - addr.sin4.sin_family == AF_INET ? (void*)&addr.sin4.sin_addr.s_addr : (void*)&addr.sin6.sin6_addr.s6_addr, + addr.sin4.sin_family == AF_INET ? reinterpret_cast(&addr.sin4.sin_addr.s_addr) : reinterpret_cast(&addr.sin6.sin6_addr.s6_addr), addr.sin4.sin_family == AF_INET ? sizeof(addr.sin4.sin_addr.s_addr) : sizeof(addr.sin6.sin6_addr.s6_addr)); dest += (addr.sin4.sin_family == AF_INET ? sizeof(addr.sin4.sin_addr.s_addr) : sizeof(addr.sin6.sin6_addr.s6_addr)); dq->getHeader()->ancount++; @@ -815,13 +909,11 @@ class SetMacAddrAction : public DNSAction public: // this action does not stop the processing SetMacAddrAction(uint16_t code) : d_code(code) - {} - DNSAction::Action operator()(DNSQuestion* dq, std::string* ruleresult) const override { - if (dq->getHeader()->arcount) { - return Action::None; - } + } + DNSAction::Action operator()(DNSQuestion* dq, std::string* ruleresult) const override + { std::string mac = getMACAddress(*dq->remote); if (mac.empty()) { return Action::None; @@ -830,21 +922,101 @@ class SetMacAddrAction : public DNSAction std::string optRData; generateEDNSOption(d_code, mac, optRData); + if (dq->getHeader()->arcount) { + bool ednsAdded = false; + bool optionAdded = false; + PacketBuffer newContent; + newContent.reserve(dq->getData().size()); + + if (!slowRewriteEDNSOptionInQueryWithRecords(dq->getData(), newContent, ednsAdded, d_code, optionAdded, true, optRData)) { + return Action::None; + } + + if (newContent.size() > dq->getMaximumSize()) { + return Action::None; + } + + dq->getMutableData() = std::move(newContent); + if (!dq->ednsAdded && ednsAdded) { + dq->ednsAdded = true; + } + + return Action::None; + } + auto& data = dq->getMutableData(); if (generateOptRR(optRData, data, dq->getMaximumSize(), g_EdnsUDPPayloadSize, 0, false)) { dq->getHeader()->arcount = htons(1); + // make sure that any EDNS sent by the backend is removed before forwarding the response to the client + dq->ednsAdded = true; } return Action::None; } std::string toString() const override { - return "add EDNS MAC (code="+std::to_string(d_code)+")"; + return "add EDNS MAC (code=" + std::to_string(d_code) + ")"; } private: uint16_t d_code{3}; }; + +class SetEDNSOptionAction : public DNSAction +{ +public: + // this action does not stop the processing + SetEDNSOptionAction(uint16_t code, const std::string& data) : d_code(code), d_data(data) + { + } + + DNSAction::Action operator()(DNSQuestion* dq, std::string* ruleresult) const override + { + std::string optRData; + generateEDNSOption(d_code, d_data, optRData); + + if (dq->getHeader()->arcount) { + bool ednsAdded = false; + bool optionAdded = false; + PacketBuffer newContent; + newContent.reserve(dq->getData().size()); + + if (!slowRewriteEDNSOptionInQueryWithRecords(dq->getData(), newContent, ednsAdded, d_code, optionAdded, true, optRData)) { + return Action::None; + } + + if (newContent.size() > dq->getMaximumSize()) { + return Action::None; + } + + dq->getMutableData() = std::move(newContent); + if (!dq->ednsAdded && ednsAdded) { + dq->ednsAdded = true; + } + + return Action::None; + } + + auto& data = dq->getMutableData(); + if (generateOptRR(optRData, data, dq->getMaximumSize(), g_EdnsUDPPayloadSize, 0, false)) { + dq->getHeader()->arcount = htons(1); + // make sure that any EDNS sent by the backend is removed before forwarding the response to the client + dq->ednsAdded = true; + } + + return Action::None; + } + + std::string toString() const override + { + return "add EDNS Option (code=" + std::to_string(d_code) + ")"; + } + +private: + uint16_t d_code; + std::string d_data; +}; + class SetNoRecurseAction : public DNSAction { public: @@ -1226,25 +1398,25 @@ class SetECSAction : public DNSAction static DnstapMessage::ProtocolType ProtocolToDNSTap(dnsdist::Protocol protocol) { - DnstapMessage::ProtocolType result; - switch (protocol) { - default: - case dnsdist::Protocol::DoUDP: - case dnsdist::Protocol::DNSCryptUDP: - result = DnstapMessage::ProtocolType::DoUDP; - break; - case dnsdist::Protocol::DoTCP: - case dnsdist::Protocol::DNSCryptTCP: - result = DnstapMessage::ProtocolType::DoTCP; - break; - case dnsdist::Protocol::DoT: - result = DnstapMessage::ProtocolType::DoT; - break; - case dnsdist::Protocol::DoH: - result = DnstapMessage::ProtocolType::DoH; - break; - } - return result; + if (protocol == dnsdist::Protocol::DoUDP) { + return DnstapMessage::ProtocolType::DoUDP; + } + else if (protocol == dnsdist::Protocol::DoTCP) { + return DnstapMessage::ProtocolType::DoTCP; + } + else if (protocol == dnsdist::Protocol::DoT) { + return DnstapMessage::ProtocolType::DoT; + } + else if (protocol == dnsdist::Protocol::DoH) { + return DnstapMessage::ProtocolType::DoH; + } + else if (protocol == dnsdist::Protocol::DNSCryptUDP) { + return DnstapMessage::ProtocolType::DNSCryptUDP; + } + else if (protocol == dnsdist::Protocol::DNSCryptTCP) { + return DnstapMessage::ProtocolType::DNSCryptTCP; + } + throw std::runtime_error("Unhandled protocol for dnstap: " + protocol.toPrettyString()); } class DnstapLogAction : public DNSAction, public boost::noncopyable @@ -1362,11 +1534,7 @@ class SetTagAction : public DNSAction } DNSAction::Action operator()(DNSQuestion* dq, std::string* ruleresult) const override { - if (!dq->qTag) { - dq->qTag = std::make_shared(); - } - - dq->qTag->insert({d_tag, d_value}); + dq->setTag(d_tag, d_value); return Action::None; } @@ -1542,11 +1710,7 @@ class SetTagResponseAction : public DNSResponseAction } DNSResponseAction::Action operator()(DNSResponse* dr, std::string* ruleresult) const override { - if (!dr->qTag) { - dr->qTag = std::make_shared(); - } - - dr->qTag->insert({d_tag, d_value}); + dr->setTag(d_tag, d_value); return Action::None; } @@ -1646,11 +1810,7 @@ class KeyValueStoreLookupAction : public DNSAction } } - if (!dq->qTag) { - dq->qTag = std::make_shared(); - } - - dq->qTag->insert({d_tag, std::move(result)}); + dq->setTag(d_tag, std::move(result)); return Action::None; } @@ -1684,11 +1844,7 @@ class KeyValueStoreRangeLookupAction : public DNSAction } } - if (!dq->qTag) { - dq->qTag = std::make_shared(); - } - - dq->qTag->insert({d_tag, std::move(result)}); + dq->setTag(d_tag, std::move(result)); return Action::None; } @@ -1955,6 +2111,10 @@ void setupLuaActions(LuaContext& luaCtx) return std::shared_ptr(new SetMacAddrAction(code)); }); + luaCtx.writeFunction("SetEDNSOptionAction", [](int code, const std::string& data) { + return std::shared_ptr(new SetEDNSOptionAction(code, data)); + }); + luaCtx.writeFunction("MacAddrAction", [](int code) { warnlog("access to MacAddrAction is deprecated and will be removed in a future version, please use SetMacAddrAction instead"); return std::shared_ptr(new SetMacAddrAction(code)); @@ -1989,6 +2149,13 @@ void setupLuaActions(LuaContext& luaCtx) return ret; }); + luaCtx.writeFunction("SpoofSVCAction", [](const std::vector>& parameters, boost::optional vars) { + auto ret = std::shared_ptr(new SpoofSVCAction(parameters)); + auto sa = std::dynamic_pointer_cast(ret); + parseResponseConfig(vars, sa->d_responseConfig); + return ret; + }); + luaCtx.writeFunction("SpoofCNAMEAction", [](const std::string& a, boost::optional vars) { auto ret = std::shared_ptr(new SpoofAction(DNSName(a))); auto sa = std::dynamic_pointer_cast(ret); diff --git a/pdns/dnsdist-lua-bindings-dnsquestion.cc b/pdns/dnsdist-lua-bindings-dnsquestion.cc index 3ddfa4415728..9d2033139ceb 100644 --- a/pdns/dnsdist-lua-bindings-dnsquestion.cc +++ b/pdns/dnsdist-lua-bindings-dnsquestion.cc @@ -74,7 +74,7 @@ void setupLuaBindingsDNSQuestion(LuaContext& luaCtx) }); luaCtx.registerFunction("getProtocol", [](const DNSQuestion& dq) { - return dnsdist::ProtocolToString(dq.getProtocol()); + return dq.getProtocol().toPrettyString(); }); luaCtx.registerFunction("sendTrap", [](const DNSQuestion& dq, boost::optional reason) { @@ -86,18 +86,11 @@ void setupLuaBindingsDNSQuestion(LuaContext& luaCtx) }); luaCtx.registerFunction("setTag", [](DNSQuestion& dq, const std::string& strLabel, const std::string& strValue) { - if(dq.qTag == nullptr) { - dq.qTag = std::make_shared(); - } - dq.qTag->insert({strLabel, strValue}); + dq.setTag(strLabel, strValue); }); luaCtx.registerFunction>)>("setTagArray", [](DNSQuestion& dq, const vector>&tags) { - if (!dq.qTag) { - dq.qTag = std::make_shared(); - } - for (const auto& tag : tags) { - dq.qTag->insert({tag.first, tag.second}); + dq.setTag(tag.first, tag.second); } }); luaCtx.registerFunction("getTag", [](const DNSQuestion& dq, const std::string& strLabel) { @@ -215,19 +208,12 @@ void setupLuaBindingsDNSQuestion(LuaContext& luaCtx) }); luaCtx.registerFunction("setTag", [](DNSResponse& dr, const std::string& strLabel, const std::string& strValue) { - if(dr.qTag == nullptr) { - dr.qTag = std::make_shared(); - } - dr.qTag->insert({strLabel, strValue}); + dr.setTag(strLabel, strValue); }); luaCtx.registerFunction>)>("setTagArray", [](DNSResponse& dr, const vector>&tags) { - if (!dr.qTag) { - dr.qTag = std::make_shared(); - } - for (const auto& tag : tags) { - dr.qTag->insert({tag.first, tag.second}); + dr.setTag(tag.first, tag.second); } }); luaCtx.registerFunction("getTag", [](const DNSResponse& dr, const std::string& strLabel) { @@ -252,7 +238,7 @@ void setupLuaBindingsDNSQuestion(LuaContext& luaCtx) }); luaCtx.registerFunction("getProtocol", [](const DNSResponse& dr) { - return dnsdist::ProtocolToString(dr.getProtocol()); + return dr.getProtocol().toPrettyString(); }); luaCtx.registerFunction("sendTrap", [](const DNSResponse& dr, boost::optional reason) { diff --git a/pdns/dnsdist-lua-bindings.cc b/pdns/dnsdist-lua-bindings.cc index 7f0568ff8816..ae44c7b63123 100644 --- a/pdns/dnsdist-lua-bindings.cc +++ b/pdns/dnsdist-lua-bindings.cc @@ -22,6 +22,7 @@ #include "config.h" #include "dnsdist.hh" #include "dnsdist-lua.hh" +#include "dnsdist-svc.hh" #include "dolog.hh" @@ -396,22 +397,91 @@ void setupLuaBindings(LuaContext& luaCtx, bool client) /* BPF Filter */ #ifdef HAVE_EBPF - luaCtx.writeFunction("newBPFFilter", [client](uint32_t maxV4, uint32_t maxV6, uint32_t maxQNames) { + using bpfFilterMapParams = boost::variant>>; + luaCtx.writeFunction("newBPFFilter", [client](bpfFilterMapParams v4Params, bpfFilterMapParams v6Params, bpfFilterMapParams qnameParams, boost::optional external) { if (client) { return std::shared_ptr(nullptr); } - return std::make_shared(maxV4, maxV6, maxQNames); + + BPFFilter::MapConfiguration v4Config, v6Config, qnameConfig; + + auto convertParamsToConfig = [](bpfFilterMapParams& params, BPFFilter::MapType type, BPFFilter::MapConfiguration& config) { + config.d_type = type; + if (params.type() == typeid(uint32_t)) { + config.d_maxItems = boost::get(params); + } + else if (params.type() == typeid(std::unordered_map>)) { + auto map = boost::get>>(params); + if (map.count("maxItems")) { + config.d_maxItems = boost::get(map.at("maxItems")); + } + if (map.count("pinnedPath")) { + config.d_pinnedPath = boost::get(map.at("pinnedPath")); + } + } + }; + + convertParamsToConfig(v4Params, BPFFilter::MapType::IPv4, v4Config); + convertParamsToConfig(v6Params, BPFFilter::MapType::IPv6, v6Config); + convertParamsToConfig(qnameParams, BPFFilter::MapType::QNames, qnameConfig); + + BPFFilter::MapFormat format = BPFFilter::MapFormat::Legacy; + if (external && *external) { + format = BPFFilter::MapFormat::WithActions; + } + + return std::make_shared(v4Config, v6Config, qnameConfig, format, external.value_or(false)); }); - luaCtx.registerFunction::*)(const ComboAddress& ca)>("block", [](std::shared_ptr bpf, const ComboAddress& ca) { + luaCtx.registerFunction::*)(const ComboAddress& ca, boost::optional action)>("block", [](std::shared_ptr bpf, const ComboAddress& ca, boost::optional action) { if (bpf) { - return bpf->block(ca); + if (!action) { + return bpf->block(ca, BPFFilter::MatchAction::Drop); + } + else { + BPFFilter::MatchAction match; + + switch (*action) { + case 0: + match = BPFFilter::MatchAction::Pass; + break; + case 1: + match = BPFFilter::MatchAction::Drop; + break; + case 2: + match = BPFFilter::MatchAction::Truncate; + break; + default: + throw std::runtime_error("Unsupported action for BPFFilter::block"); + } + return bpf->block(ca, match); + } } }); - luaCtx.registerFunction::*)(const DNSName& qname, boost::optional qtype)>("blockQName", [](std::shared_ptr bpf, const DNSName& qname, boost::optional qtype) { + luaCtx.registerFunction::*)(const DNSName& qname, boost::optional qtype, boost::optional action)>("blockQName", [](std::shared_ptr bpf, const DNSName& qname, boost::optional qtype, boost::optional action) { if (bpf) { - return bpf->block(qname, qtype ? *qtype : 255); + if (!action) { + return bpf->block(qname, BPFFilter::MatchAction::Drop, qtype.value_or(255)); + } + else { + BPFFilter::MatchAction match; + + switch (*action) { + case 0: + match = BPFFilter::MatchAction::Pass; + break; + case 1: + match = BPFFilter::MatchAction::Drop; + break; + case 2: + match = BPFFilter::MatchAction::Truncate; + break; + default: + throw std::runtime_error("Unsupported action for BPFFilter::blockQName"); + } + return bpf->block(qname, match, qtype.value_or(255)); + } } }); @@ -534,4 +604,16 @@ void setupLuaBindings(LuaContext& luaCtx, bool client) } return std::make_shared(regex, status, PacketBuffer(content.begin(), content.end()), headers); }); + + luaCtx.writeFunction("newSVCRecordParameters", [](uint16_t priority, const std::string& target, boost::optional additionalParameters) + { + SVCRecordParameters parameters; + if (additionalParameters) { + parameters = parseSVCParameters(*additionalParameters); + } + parameters.priority = priority; + parameters.target = DNSName(target); + + return parameters; + }); } diff --git a/pdns/dnsdist-lua-inspection.cc b/pdns/dnsdist-lua-inspection.cc index 1853e0fc0fc9..451682a97ea6 100644 --- a/pdns/dnsdist-lua-inspection.cc +++ b/pdns/dnsdist-lua-inspection.cc @@ -22,6 +22,7 @@ #include "dnsdist.hh" #include "dnsdist-lua.hh" #include "dnsdist-dynblocks.hh" +#include "dnsdist-nghttp2.hh" #include "dnsdist-rings.hh" #include "dnsdist-tcp.hh" @@ -61,7 +62,7 @@ static std::unordered_map>> g vector> rcounts; rcounts.reserve(counts.size()); for(const auto& c : counts) - rcounts.push_back(make_pair(c.second, c.first.makeLowerCase())); + rcounts.emplace_back(c.second, c.first.makeLowerCase()); sort(rcounts.begin(), rcounts.end(), [](const decltype(rcounts)::value_type& a, const decltype(rcounts)::value_type& b) { @@ -149,7 +150,7 @@ static vector > > getRespRi continue; e["qname"]=c.name.toString(); e["rcode"]=std::to_string(c.dh.rcode); - ret.push_back(std::make_pair(count,e)); + ret.emplace_back(count, e); count++; } } @@ -250,7 +251,7 @@ void setupLuaInspection(LuaContext& luaCtx) vector> rcounts; rcounts.reserve(counts.size()); for(const auto& c : counts) - rcounts.push_back(make_pair(c.second, c.first)); + rcounts.emplace_back(c.second, c.first); sort(rcounts.begin(), rcounts.end(), [](const decltype(rcounts)::value_type& a, const decltype(rcounts)::value_type& b) { @@ -295,7 +296,7 @@ void setupLuaInspection(LuaContext& luaCtx) vector> rcounts; rcounts.reserve(counts.size()); for(const auto& c : counts) - rcounts.push_back(make_pair(c.second, c.first.makeLowerCase())); + rcounts.emplace_back(c.second, c.first.makeLowerCase()); sort(rcounts.begin(), rcounts.end(), [](const decltype(rcounts)::value_type& a, const decltype(rcounts)::value_type& b) { @@ -454,8 +455,8 @@ void setupLuaInspection(LuaContext& luaCtx) std::multimap out; - boost::format fmt("%-7.1f %-47s %-12s %-5d %-25s %-5s %-6.1f %-2s %-2s %-2s %-s\n"); - g_outputBuffer+= (fmt % "Time" % "Client" % "Server" % "ID" % "Name" % "Type" % "Lat." % "TC" % "RD" % "AA" % "Rcode").str(); + boost::format fmt("%-7.1f %-47s %-12s %-12s %-5d %-25s %-5s %-6.1f %-2s %-2s %-2s %-s\n"); + g_outputBuffer+= (fmt % "Time" % "Client" % "Protocol" % "Server" % "ID" % "Name" % "Type" % "Lat." % "TC" % "RD" % "AA" % "Rcode").str(); if(msec==-1) { for(const auto& c : qr) { @@ -477,7 +478,7 @@ void setupLuaInspection(LuaContext& luaCtx) if (c.dh.opcode != 0) { extra = " (" + Opcode::to_s(c.dh.opcode) + ")"; } - out.insert(make_pair(c.when, (fmt % DiffTime(now, c.when) % c.requestor.toStringWithPort() % "" % htons(c.dh.id) % c.name.toString() % qt.toString() % "" % (c.dh.tc ? "TC" : "") % (c.dh.rd? "RD" : "") % (c.dh.aa? "AA" : "") % ("Question" + extra)).str() )) ; + out.emplace(c.when, (fmt % DiffTime(now, c.when) % c.requestor.toStringWithPort() % dnsdist::Protocol(c.protocol).toString() % "" % htons(c.dh.id) % c.name.toString() % qt.toString() % "" % (c.dh.tc ? "TC" : "") % (c.dh.rd ? "RD" : "") % (c.dh.aa ? "AA" : "") % ("Question" + extra)).str()); if(limit && *limit==++num) break; @@ -515,10 +516,10 @@ void setupLuaInspection(LuaContext& luaCtx) } if (c.usec != std::numeric_limits::max()) { - out.insert(make_pair(c.when, (fmt % DiffTime(now, c.when) % c.requestor.toStringWithPort() % c.ds.toStringWithPort() % htons(c.dh.id) % c.name.toString() % qt.toString() % (c.usec/1000.0) % (c.dh.tc ? "TC" : "") % (c.dh.rd? "RD" : "") % (c.dh.aa? "AA" : "") % (RCode::to_s(c.dh.rcode) + extra)).str() )) ; + out.emplace(c.when, (fmt % DiffTime(now, c.when) % c.requestor.toStringWithPort() % dnsdist::Protocol(c.protocol).toString() % c.ds.toStringWithPort() % htons(c.dh.id) % c.name.toString() % qt.toString() % (c.usec / 1000.0) % (c.dh.tc ? "TC" : "") % (c.dh.rd ? "RD" : "") % (c.dh.aa ? "AA" : "") % (RCode::to_s(c.dh.rcode) + extra)).str()); } else { - out.insert(make_pair(c.when, (fmt % DiffTime(now, c.when) % c.requestor.toStringWithPort() % c.ds.toStringWithPort() % htons(c.dh.id) % c.name.toString() % qt.toString() % "T.O" % (c.dh.tc ? "TC" : "") % (c.dh.rd? "RD" : "") % (c.dh.aa? "AA" : "") % (RCode::to_s(c.dh.rcode) + extra)).str() )) ; + out.emplace(c.when, (fmt % DiffTime(now, c.when) % c.requestor.toStringWithPort() % dnsdist::Protocol(c.protocol).toString() % c.ds.toStringWithPort() % htons(c.dh.id) % c.name.toString() % qt.toString() % "T.O" % (c.dh.tc ? "TC" : "") % (c.dh.rd ? "RD" : "") % (c.dh.aa ? "AA" : "") % (RCode::to_s(c.dh.rcode) + extra)).str()); } if (limit && *limit == ++num) { @@ -610,8 +611,8 @@ void setupLuaInspection(LuaContext& luaCtx) ret << endl; ret << "Backends:" << endl; - fmt = boost::format("%-3d %-20.20s %-20.20s %-20d %-20d %-25d %-20d %-20d %-20d %-20d %-20d %-20d %-20d %-20d %-20f %-20f"); - ret << (fmt % "#" % "Name" % "Address" % "Connections" % " Max concurrent conn" % "Died sending query" % "Died reading response" % "Gave up" % "Read timeouts" % "Write timeouts" % "Connect timeouts" % "Total connections" % "Reused connections" % "TLS resumptions" % "Avg queries/conn" % "Avg duration") << endl; + fmt = boost::format("%-3d %-20.20s %-20.20s %-20d %-20d %-25d %-25d %-20d %-20d %-20d %-20d %-20d %-20d %-20d %-20f %-20f"); + ret << (fmt % "#" % "Name" % "Address" % "Connections" % "Max concurrent conn" % "Died sending query" % "Died reading response" % "Gave up" % "Read timeouts" % "Write timeouts" % "Connect timeouts" % "Total connections" % "Reused connections" % "TLS resumptions" % "Avg queries/conn" % "Avg duration") << endl; auto states = g_dstates.getLocal(); counter = 0; @@ -660,11 +661,16 @@ void setupLuaInspection(LuaContext& luaCtx) g_tcpStatesDumpRequested += g_tcpclientthreads->getThreadsCount(); }); + luaCtx.writeFunction("requestDoHStatesDump", [] { + setLuaNoSideEffect(); + g_dohStatesDumpRequested += g_dohClientThreads->getThreadsCount(); + }); + luaCtx.writeFunction("dumpStats", [] { setLuaNoSideEffect(); vector leftcolumn, rightcolumn; - boost::format fmt("%-23s\t%+11s"); + boost::format fmt("%-35s\t%+11s"); g_outputBuffer.clear(); auto entries = g_stats.entries; sort(entries.begin(), entries.end(), @@ -688,7 +694,7 @@ void setupLuaInspection(LuaContext& luaCtx) } auto leftiter=leftcolumn.begin(), rightiter=rightcolumn.begin(); - boost::format clmn("%|0t|%1% %|39t|%2%\n"); + boost::format clmn("%|0t|%1% %|51t|%2%\n"); for(;leftiter != leftcolumn.end() || rightiter != rightcolumn.end();) { string lentry, rentry; @@ -790,6 +796,23 @@ void setupLuaInspection(LuaContext& luaCtx) group->setQTypeRate(qtype, rate, warningRate ? *warningRate : 0, seconds, reason, blockDuration, action ? *action : DNSAction::Action::None); } }); + luaCtx.registerFunction::*)(uint8_t, uint8_t, uint8_t)>("setMasks", [](std::shared_ptr& group, uint8_t v4, uint8_t v6, uint8_t port) { + if (group) { + if (v4 > 32) { + throw std::runtime_error("Trying to set an invalid IPv4 mask (" + std::to_string(v4) + ") to a Dynamic Block object"); + } + if (v6 > 128) { + throw std::runtime_error("Trying to set an invalid IPv6 mask (" + std::to_string(v6) + ") to a Dynamic Block object"); + } + if (port > 16) { + throw std::runtime_error("Trying to set an invalid port mask (" + std::to_string(port) + ") to a Dynamic Block object"); + } + if (port > 0 && v4 != 32) { + throw std::runtime_error("Setting a non-zero port mask for Dynamic Blocks while only considering parts of IPv4 addresses does not make sense"); + } + group->setMasks(v4, v6, port); + } + }); luaCtx.registerFunction::*)(boost::variant>, NetmaskGroup>)>("excludeRange", [](std::shared_ptr& group, boost::variant>, NetmaskGroup> ranges) { if (ranges.type() == typeid(std::vector>)) { for (const auto& range : *boost::get>>(&ranges)) { diff --git a/pdns/dnsdist-lua-rules.cc b/pdns/dnsdist-lua-rules.cc index 12e44a042c88..19eec4ca6622 100644 --- a/pdns/dnsdist-lua-rules.cc +++ b/pdns/dnsdist-lua-rules.cc @@ -568,6 +568,10 @@ void setupLuaRules(LuaContext& luaCtx) return std::shared_ptr(new PoolAvailableRule(poolname)); }); + luaCtx.writeFunction("PoolOutstandingRule", [](std::string poolname, size_t limit) { + return std::shared_ptr(new PoolOutstandingRule(poolname, limit)); + }); + luaCtx.registerFunction::*)()>("clear", [](std::shared_ptr tisr) { tisr->clear(); }); diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index 69663e5191b8..fde21a9fccb4 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -40,6 +40,7 @@ #ifdef LUAJIT_VERSION #include "dnsdist-lua-ffi.hh" #endif /* LUAJIT_VERSION */ +#include "dnsdist-nghttp2.hh" #include "dnsdist-proxy-protocol.hh" #include "dnsdist-rings.hh" #include "dnsdist-secpoll.hh" @@ -76,14 +77,14 @@ static bool g_included{false}; has done so before on this invocation, this call won't be part of delta() output */ void setLuaNoSideEffect() { - if(g_noLuaSideEffect==false) // there has been a side effect already + if (g_noLuaSideEffect == false) // there has been a side effect already return; - g_noLuaSideEffect=true; + g_noLuaSideEffect = true; } void setLuaSideEffect() { - g_noLuaSideEffect=false; + g_noLuaSideEffect = false; } bool getLuaNoSideEffect() @@ -99,7 +100,7 @@ void resetLuaSideEffect() g_noLuaSideEffect = boost::logic::indeterminate; } -typedef std::unordered_map >, std::vector >, std::map > > localbind_t; +typedef std::unordered_map>, std::vector>, std::map>> localbind_t; static void parseLocalBindVars(boost::optional vars, bool& reusePort, int& tcpFastOpenQueueSize, std::string& interface, std::set& cpus, int& tcpListenQueueSize, size_t& maxInFlightQueriesPerConnection, size_t& tcpMaxConcurrentConnections) { @@ -123,7 +124,7 @@ static void parseLocalBindVars(boost::optional vars, bool& reusePor interface = boost::get((*vars)["interface"]); } if (vars->count("cpus")) { - for (const auto& cpu : boost::get>>((*vars)["cpus"])) { + for (const auto& cpu : boost::get>>((*vars)["cpus"])) { cpus.insert(cpu.second); } } @@ -131,33 +132,32 @@ static void parseLocalBindVars(boost::optional vars, bool& reusePor } #if defined(HAVE_DNS_OVER_TLS) || defined(HAVE_DNS_OVER_HTTPS) -static bool loadTLSCertificateAndKeys(const std::string& context, std::vector>& pairs, boost::variant>> certFiles, boost::variant>> keyFiles) +static bool loadTLSCertificateAndKeys(const std::string& context, std::vector>& pairs, boost::variant>> certFiles, boost::variant>> keyFiles) { if (certFiles.type() == typeid(std::string) && keyFiles.type() == typeid(std::string)) { auto certFile = boost::get(certFiles); auto keyFile = boost::get(keyFiles); pairs.clear(); - pairs.push_back({certFile, keyFile}); + pairs.emplace_back(certFile, keyFile); } - else if (certFiles.type() == typeid(std::vector>) && keyFiles.type() == typeid(std::vector>)) - { - auto certFilesVect = boost::get>>(certFiles); - auto keyFilesVect = boost::get>>(keyFiles); + else if (certFiles.type() == typeid(std::vector>) && keyFiles.type() == typeid(std::vector>)) { + auto certFilesVect = boost::get>>(certFiles); + auto keyFilesVect = boost::get>>(keyFiles); if (certFilesVect.size() == keyFilesVect.size()) { pairs.clear(); for (size_t idx = 0; idx < certFilesVect.size(); idx++) { - pairs.push_back({certFilesVect.at(idx).second, keyFilesVect.at(idx).second}); + pairs.emplace_back(certFilesVect.at(idx).second, keyFilesVect.at(idx).second); } } else { errlog("Error, mismatching number of certificates and keys in call to %s()!", context); - g_outputBuffer="Error, mismatching number of certificates and keys in call to " + context + "()!"; + g_outputBuffer = "Error, mismatching number of certificates and keys in call to " + context + "()!"; return false; } } else { errlog("Error, mismatching number of certificates and keys in call to %s()!", context); - g_outputBuffer="Error, mismatching number of certificates and keys in call to " + context + "()!"; + g_outputBuffer = "Error, mismatching number of certificates and keys in call to " + context + "()!"; return false; } @@ -208,7 +208,7 @@ static void parseTLSConfig(TLSConfig& config, const std::string& context, boost: auto value = boost::get((*vars)["numberOfStoredSessions"]); if (value < 0) { errlog("Invalid value '%d' for %s() parameter 'numberOfStoredSessions', should be >= 0, dismissing", value, context); - g_outputBuffer="Invalid value '" + std::to_string(value) + "' for " + context + "() parameter 'numberOfStoredSessions', should be >= 0, dismissing"; + g_outputBuffer = "Invalid value '" + std::to_string(value) + "' for " + context + "() parameter 'numberOfStoredSessions', should be >= 0, dismissing"; } config.d_maxStoredSessions = value; } @@ -240,389 +240,430 @@ static void parseTLSConfig(TLSConfig& config, const std::string& context, boost: #endif // defined(HAVE_DNS_OVER_TLS) || defined(HAVE_DNS_OVER_HTTPS) +static void checkParameterBound(const std::string& parameter, uint64_t value, size_t max = std::numeric_limits::max()) +{ + if (value > std::numeric_limits::max()) { + throw std::runtime_error("The value passed to " + parameter + " is too large, the maximum is " + std::to_string(max)); + } +} + static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) { - typedef std::unordered_map >, DownstreamState::checkfunc_t > > newserver_t; + typedef std::unordered_map>, DownstreamState::checkfunc_t>> newserver_t; luaCtx.writeFunction("inClientStartup", [client]() { - return client && !g_configurationDone; + return client && !g_configurationDone; }); luaCtx.writeFunction("inConfigCheck", [configCheck]() { - return !configCheck; + return !configCheck; }); luaCtx.writeFunction("newServer", - [client, configCheck](boost::variant pvars, boost::optional qps) { - setLuaSideEffect(); - - std::shared_ptr ret = std::make_shared(ComboAddress(), ComboAddress(), 0, std::string(), 1, !(client || configCheck)); - newserver_t vars; - - ComboAddress serverAddr; - std::string serverAddressStr; - if(auto addrStr = boost::get(&pvars)) { - serverAddressStr = *addrStr; - if(qps) { - vars["qps"] = std::to_string(*qps); - } - } else { - vars = boost::get(pvars); - serverAddressStr = boost::get(vars["address"]); - } + [client, configCheck](boost::variant pvars, boost::optional qps) { + setLuaSideEffect(); - try { - serverAddr = ComboAddress(serverAddressStr, 53); - } - catch(const PDNSException& e) { - g_outputBuffer="Error creating new server: "+string(e.reason); - errlog("Error creating new server with address %s: %s", serverAddressStr, e.reason); - return ret; - } - catch(std::exception& e) { - g_outputBuffer="Error creating new server: "+string(e.what()); - errlog("Error creating new server with address %s: %s", serverAddressStr, e.what()); - return ret; - } - - if(IsAnyAddress(serverAddr)) { - g_outputBuffer="Error creating new server: invalid address for a downstream server."; - errlog("Error creating new server: %s is not a valid address for a downstream server", serverAddressStr); - return ret; - } - - ComboAddress sourceAddr; - std::string sourceItfName; - unsigned int sourceItf = 0; - size_t numberOfSockets = 1; - std::set cpus; - - if(vars.count("source")) { - /* handle source in the following forms: - - v4 address ("192.0.2.1") - - v6 address ("2001:DB8::1") - - interface name ("eth0") - - v4 address and interface name ("192.0.2.1@eth0") - - v6 address and interface name ("2001:DB8::1@eth0") - */ - const string source = boost::get(vars["source"]); - bool parsed = false; - std::string::size_type pos = source.find("@"); - if (pos == std::string::npos) { - /* no '@', try to parse that as a valid v4/v6 address */ - try { - sourceAddr = ComboAddress(source); - parsed = true; - } - catch(...) - { - } - } + newserver_t vars; + + ComboAddress serverAddr; + std::string serverAddressStr; + if (auto addrStr = boost::get(&pvars)) { + serverAddressStr = *addrStr; + if (qps) { + vars["qps"] = std::to_string(*qps); + } + } + else { + vars = boost::get(pvars); + serverAddressStr = boost::get(vars["address"]); + } + + try { + serverAddr = ComboAddress(serverAddressStr, 53); + } + catch (const PDNSException& e) { + g_outputBuffer = "Error creating new server: " + string(e.reason); + errlog("Error creating new server with address %s: %s", serverAddressStr, e.reason); + return std::shared_ptr(); + } + catch (const std::exception& e) { + g_outputBuffer = "Error creating new server: " + string(e.what()); + errlog("Error creating new server with address %s: %s", serverAddressStr, e.what()); + return std::shared_ptr(); + } + + if (IsAnyAddress(serverAddr)) { + g_outputBuffer = "Error creating new server: invalid address for a downstream server."; + errlog("Error creating new server: %s is not a valid address for a downstream server", serverAddressStr); + return std::shared_ptr(); + } + + ComboAddress sourceAddr; + std::string sourceItfName; + unsigned int sourceItf = 0; + size_t numberOfSockets = 1; + std::set cpus; + + if (vars.count("source")) { + /* handle source in the following forms: + - v4 address ("192.0.2.1") + - v6 address ("2001:DB8::1") + - interface name ("eth0") + - v4 address and interface name ("192.0.2.1@eth0") + - v6 address and interface name ("2001:DB8::1@eth0") + */ + const string source = boost::get(vars["source"]); + bool parsed = false; + std::string::size_type pos = source.find("@"); + if (pos == std::string::npos) { + /* no '@', try to parse that as a valid v4/v6 address */ + try { + sourceAddr = ComboAddress(source); + parsed = true; + } + catch (...) { + } + } - if (parsed == false) - { - /* try to parse as interface name, or v4/v6@itf */ - sourceItfName = source.substr(pos == std::string::npos ? 0 : pos + 1); - unsigned int itfIdx = if_nametoindex(sourceItfName.c_str()); - - if (itfIdx != 0) { - if (pos == 0 || pos == std::string::npos) { - /* "eth0" or "@eth0" */ - sourceItf = itfIdx; - } - else { - /* "192.0.2.1@eth0" */ - sourceAddr = ComboAddress(source.substr(0, pos)); - sourceItf = itfIdx; - } + if (parsed == false) { + /* try to parse as interface name, or v4/v6@itf */ + sourceItfName = source.substr(pos == std::string::npos ? 0 : pos + 1); + unsigned int itfIdx = if_nametoindex(sourceItfName.c_str()); + + if (itfIdx != 0) { + if (pos == 0 || pos == std::string::npos) { + /* "eth0" or "@eth0" */ + sourceItf = itfIdx; + } + else { + /* "192.0.2.1@eth0" */ + sourceAddr = ComboAddress(source.substr(0, pos)); + sourceItf = itfIdx; + } #ifdef SO_BINDTODEVICE - /* we need to retain CAP_NET_RAW to be able to set SO_BINDTODEVICE in the health checks */ - g_capabilitiesToRetain.insert("CAP_NET_RAW"); + /* we need to retain CAP_NET_RAW to be able to set SO_BINDTODEVICE in the health checks */ + g_capabilitiesToRetain.insert("CAP_NET_RAW"); #endif - } - else - { - warnlog("Dismissing source %s because '%s' is not a valid interface name", source, sourceItfName); - } - } - } + } + else { + warnlog("Dismissing source %s because '%s' is not a valid interface name", source, sourceItfName); + } + } + } - if (vars.count("sockets")) { - numberOfSockets = std::stoul(boost::get(vars["sockets"])); - if (numberOfSockets == 0) { - warnlog("Dismissing invalid number of sockets '%s', using 1 instead", boost::get(vars["sockets"])); - numberOfSockets = 1; - } - } + if (vars.count("sockets")) { + numberOfSockets = std::stoul(boost::get(vars["sockets"])); + if (numberOfSockets == 0) { + warnlog("Dismissing invalid number of sockets '%s', using 1 instead", boost::get(vars["sockets"])); + numberOfSockets = 1; + } + } - // create but don't connect the socket in client or check-config modes - ret=std::make_shared(serverAddr, sourceAddr, sourceItf, sourceItfName, numberOfSockets, !(client || configCheck)); - if (!(client || configCheck)) { - infolog("Added downstream server %s", serverAddr.toStringWithPort()); - } + // create but don't connect the socket in client or check-config modes + auto ret = std::make_shared(serverAddr, sourceAddr, sourceItf, sourceItfName); + if (!(client || configCheck)) { + infolog("Added downstream server %s", serverAddr.toStringWithPort()); + } - if(vars.count("qps")) { - int qpsVal=std::stoi(boost::get(vars["qps"])); - ret->qps=QPSLimiter(qpsVal, qpsVal); - } + if (vars.count("qps")) { + int qpsVal = std::stoi(boost::get(vars["qps"])); + ret->qps = QPSLimiter(qpsVal, qpsVal); + } - if(vars.count("order")) { - ret->order=std::stoi(boost::get(vars["order"])); - } + if (vars.count("order")) { + ret->order = std::stoi(boost::get(vars["order"])); + } - if(vars.count("weight")) { - try { - int weightVal=std::stoi(boost::get(vars["weight"])); + if (vars.count("weight")) { + try { + int weightVal = std::stoi(boost::get(vars["weight"])); - if(weightVal < 1) { - errlog("Error creating new server: downstream weight value must be greater than 0."); - return ret; - } + if (weightVal < 1) { + errlog("Error creating new server: downstream weight value must be greater than 0."); + return ret; + } - ret->setWeight(weightVal); - } - catch(std::exception& e) { - // std::stoi will throw an exception if the string isn't in a value int range - errlog("Error creating new server: downstream weight value must be between %s and %s", 1, std::numeric_limits::max()); - return ret; - } - } + ret->setWeight(weightVal); + } + catch (const std::exception& e) { + // std::stoi will throw an exception if the string isn't in a value int range + errlog("Error creating new server: downstream weight value must be between %s and %s", 1, std::numeric_limits::max()); + return ret; + } + } - if(vars.count("retries")) { - ret->retries=std::stoi(boost::get(vars["retries"])); - } + if (vars.count("retries")) { + ret->d_retries = std::stoi(boost::get(vars["retries"])); + } - if(vars.count("checkInterval")) { - ret->checkInterval=static_cast(std::stoul(boost::get(vars["checkInterval"]))); - } + if (vars.count("checkInterval")) { + ret->checkInterval = static_cast(std::stoul(boost::get(vars["checkInterval"]))); + } - if(vars.count("tcpConnectTimeout")) { - ret->tcpConnectTimeout=std::stoi(boost::get(vars["tcpConnectTimeout"])); - } + if (vars.count("tcpConnectTimeout")) { + ret->tcpConnectTimeout = std::stoi(boost::get(vars["tcpConnectTimeout"])); + } - if(vars.count("tcpSendTimeout")) { - ret->tcpSendTimeout=std::stoi(boost::get(vars["tcpSendTimeout"])); - } + if (vars.count("tcpSendTimeout")) { + ret->tcpSendTimeout = std::stoi(boost::get(vars["tcpSendTimeout"])); + } - if(vars.count("tcpRecvTimeout")) { - ret->tcpRecvTimeout=std::stoi(boost::get(vars["tcpRecvTimeout"])); - } + if (vars.count("tcpRecvTimeout")) { + ret->tcpRecvTimeout = std::stoi(boost::get(vars["tcpRecvTimeout"])); + } - if(vars.count("tcpFastOpen")) { - bool fastOpen = boost::get(vars["tcpFastOpen"]); - if (fastOpen) { + if (vars.count("tcpFastOpen")) { + bool fastOpen = boost::get(vars["tcpFastOpen"]); + if (fastOpen) { #ifdef MSG_FASTOPEN - ret->tcpFastOpen=true; + ret->tcpFastOpen = true; #else warnlog("TCP Fast Open has been configured on downstream server %s but is not supported", boost::get(vars["address"])); #endif - } - } - - if(vars.count("maxInFlight")) { - ret->d_maxInFlightQueriesPerConn = std::stoi(boost::get(vars["maxInFlight"])); - } - - if(vars.count("name")) { - ret->setName(boost::get(vars["name"])); - } + } + } - if (vars.count("id")) { - ret->setId(boost::lexical_cast(boost::get(vars["id"]))); - } + if (vars.count("maxInFlight")) { + ret->d_maxInFlightQueriesPerConn = std::stoi(boost::get(vars["maxInFlight"])); + } - if(vars.count("checkName")) { - ret->checkName=DNSName(boost::get(vars["checkName"])); - } + if (vars.count("name")) { + ret->setName(boost::get(vars["name"])); + } - if(vars.count("checkType")) { - ret->checkType=boost::get(vars["checkType"]); - } + if (vars.count("id")) { + ret->setId(boost::lexical_cast(boost::get(vars["id"]))); + } - if(vars.count("checkClass")) { - ret->checkClass=std::stoi(boost::get(vars["checkClass"])); - } + if (vars.count("checkName")) { + ret->checkName = DNSName(boost::get(vars["checkName"])); + } - if(vars.count("checkFunction")) { - ret->checkFunction = boost::get(vars["checkFunction"]); - } + if (vars.count("checkType")) { + ret->checkType = boost::get(vars["checkType"]); + } - if(vars.count("checkTimeout")) { - ret->checkTimeout = std::stoi(boost::get(vars["checkTimeout"])); - } + if (vars.count("checkClass")) { + ret->checkClass = std::stoi(boost::get(vars["checkClass"])); + } - if (vars.count("checkTCP")) { - ret->d_tcpCheck = boost::get(vars.at("checkTCP")); - } + if (vars.count("checkFunction")) { + ret->checkFunction = boost::get(vars["checkFunction"]); + } - if(vars.count("setCD")) { - ret->setCD=boost::get(vars["setCD"]); - } + if (vars.count("checkTimeout")) { + ret->checkTimeout = std::stoi(boost::get(vars["checkTimeout"])); + } - if(vars.count("mustResolve")) { - ret->mustResolve=boost::get(vars["mustResolve"]); - } + if (vars.count("checkTCP")) { + ret->d_tcpCheck = boost::get(vars.at("checkTCP")); + } - if(vars.count("useClientSubnet")) { - ret->useECS=boost::get(vars["useClientSubnet"]); - } + if (vars.count("setCD")) { + ret->setCD = boost::get(vars["setCD"]); + } - if(vars.count("useProxyProtocol")) { - ret->useProxyProtocol = boost::get(vars["useProxyProtocol"]); - } + if (vars.count("mustResolve")) { + ret->mustResolve = boost::get(vars["mustResolve"]); + } - if(vars.count("disableZeroScope")) { - ret->disableZeroScope=boost::get(vars["disableZeroScope"]); - } + if (vars.count("useClientSubnet")) { + ret->useECS = boost::get(vars["useClientSubnet"]); + } - if(vars.count("ipBindAddrNoPort")) { - ret->ipBindAddrNoPort=boost::get(vars["ipBindAddrNoPort"]); - } + if (vars.count("useProxyProtocol")) { + ret->useProxyProtocol = boost::get(vars["useProxyProtocol"]); + } - if(vars.count("addXPF")) { - ret->xpfRRCode=std::stoi(boost::get(vars["addXPF"])); - } + if (vars.count("disableZeroScope")) { + ret->disableZeroScope = boost::get(vars["disableZeroScope"]); + } - if(vars.count("maxCheckFailures")) { - ret->maxCheckFailures=std::stoi(boost::get(vars["maxCheckFailures"])); - } + if (vars.count("ipBindAddrNoPort")) { + ret->ipBindAddrNoPort = boost::get(vars["ipBindAddrNoPort"]); + } - if(vars.count("rise")) { - ret->minRiseSuccesses=std::stoi(boost::get(vars["rise"])); - } + if (vars.count("addXPF")) { + ret->xpfRRCode = std::stoi(boost::get(vars["addXPF"])); + } - if(vars.count("reconnectOnUp")) { - ret->reconnectOnUp=boost::get(vars["reconnectOnUp"]); - } + if (vars.count("maxCheckFailures")) { + ret->maxCheckFailures = std::stoi(boost::get(vars["maxCheckFailures"])); + } - if(vars.count("cpus")) { - for (const auto& cpu : boost::get>>(vars["cpus"])) { - cpus.insert(std::stoi(cpu.second)); - } - } + if (vars.count("rise")) { + ret->minRiseSuccesses = std::stoi(boost::get(vars["rise"])); + } - if (vars.count("tcpOnly")) { - ret->d_tcpOnly = boost::get(vars.at("tcpOnly")); - } + if (vars.count("reconnectOnUp")) { + ret->reconnectOnUp = boost::get(vars["reconnectOnUp"]); + } - if (vars.count("tls")) { - TLSContextParameters tlsParams; - std::string ciphers; - std::string ciphers13; + if (vars.count("cpus")) { + for (const auto& cpu : boost::get>>(vars["cpus"])) { + cpus.insert(std::stoi(cpu.second)); + } + } - tlsParams.d_provider = boost::get(vars.at("tls")); + if (vars.count("tcpOnly")) { + ret->d_tcpOnly = boost::get(vars.at("tcpOnly")); + } - if (vars.count("ciphers")) { - tlsParams.d_ciphers = boost::get(vars.at("ciphers")); - } - if (vars.count("ciphers13")) { - tlsParams.d_ciphers13 = boost::get(vars.at("ciphers13")); - } - if (vars.count("caStore")) { - tlsParams.d_caStore = boost::get(vars.at("caStore")); - } - if (vars.count("validateCertificates")) { - tlsParams.d_validateCertificates = boost::get(vars.at("validateCertificates")); - } - if (vars.count("subjectName")) { - ret->d_tlsSubjectName = boost::get(vars.at("subjectName")); - } + if (vars.count("tls")) { + TLSContextParameters tlsParams; + std::string ciphers; + std::string ciphers13; - ret->d_tlsCtx = getTLSContext(tlsParams); - } + tlsParams.d_provider = boost::get(vars.at("tls")); - /* this needs to be done _AFTER_ the order has been set, - since the server are kept ordered inside the pool */ - auto localPools = g_pools.getCopy(); - if(vars.count("pool")) { - if(auto* pool = boost::get(&vars["pool"])) { - ret->pools.insert(*pool); - } - else { - auto pools = boost::get > >(vars["pool"]); - for(auto& p : pools) { - ret->pools.insert(p.second); - } - } - for(const auto& poolName: ret->pools) { - addServerToPool(localPools, poolName, ret); - } - } - else { - addServerToPool(localPools, "", ret); - } - g_pools.setState(localPools); + if (vars.count("ciphers")) { + tlsParams.d_ciphers = boost::get(vars.at("ciphers")); + } + if (vars.count("ciphers13")) { + tlsParams.d_ciphers13 = boost::get(vars.at("ciphers13")); + } + if (vars.count("caStore")) { + tlsParams.d_caStore = boost::get(vars.at("caStore")); + } + if (vars.count("validateCertificates")) { + tlsParams.d_validateCertificates = boost::get(vars.at("validateCertificates")); + } + if (vars.count("releaseBuffers")) { + tlsParams.d_releaseBuffers = boost::get(vars.at("releaseBuffers")); + } + if (vars.count("enableRenegotiation")) { + tlsParams.d_enableRenegotiation = boost::get(vars.at("enableRenegotiation")); + } + if (vars.count("subjectName")) { + ret->d_tlsSubjectName = boost::get(vars.at("subjectName")); + } - if (ret->connected) { - ret->threadStarted.test_and_set(); + ret->d_tlsCtx = getTLSContext(tlsParams); + + if (vars.count("dohPath")) { +#ifdef HAVE_NGHTTP2 + ret->d_dohPath = boost::get(vars.at("dohPath")); + if (ret->d_tlsCtx) { + setupDoHClientProtocolNegotiation(ret->d_tlsCtx); + } + + if (g_configurationDone && g_outgoingDoHWorkerThreads && *g_outgoingDoHWorkerThreads == 0) { + throw std::runtime_error("Error: setOutgoingDoHWorkerThreads() is set to 0 so no outgoing DoH worker thread is available to serve queries"); + } + + if (!g_outgoingDoHWorkerThreads || *g_outgoingDoHWorkerThreads == 0) { + g_outgoingDoHWorkerThreads = 1; + } + + if (vars.count("addXForwardedHeaders")) { + ret->d_addXForwardedHeaders = boost::get(vars.at("addXForwardedHeaders")); + } +#else /* HAVE_NGHTTP2 */ + throw std::runtime_error("Outgoing DNS over HTTPS support requested (via 'dohPath' on newServer()) but nghttp2 support is not available"); +#endif /* HAVE_NGHTTP2 */ + } + else { + setupDoTProtocolNegotiation(ret->d_tlsCtx); + } + } - if (g_launchWork) { - g_launchWork->push_back([ret,cpus]() { - ret->tid = thread(responderThread, ret); - if (!cpus.empty()) { - mapThreadToCPUList(ret->tid.native_handle(), cpus); - } - }); - } - else { - ret->tid = thread(responderThread, ret); - if (!cpus.empty()) { - mapThreadToCPUList(ret->tid.native_handle(), cpus); - } - } - } + if (!ret->isTCPOnly() && !(client || configCheck)) { + if (!IsAnyAddress(ret->remote)) { + ret->connectUDPSockets(numberOfSockets); + } + } + + /* this needs to be done _AFTER_ the order has been set, + since the server are kept ordered inside the pool */ + auto localPools = g_pools.getCopy(); + if (vars.count("pool")) { + if (auto* pool = boost::get(&vars["pool"])) { + ret->pools.insert(*pool); + } + else { + auto pools = boost::get>>(vars["pool"]); + for (auto& p : pools) { + ret->pools.insert(p.second); + } + } + for (const auto& poolName : ret->pools) { + addServerToPool(localPools, poolName, ret); + } + } + else { + addServerToPool(localPools, "", ret); + } + g_pools.setState(localPools); + + if (ret->connected) { + ret->threadStarted.test_and_set(); + + if (g_launchWork) { + g_launchWork->push_back([ret, cpus]() { + ret->tid = thread(responderThread, ret); + if (!cpus.empty()) { + mapThreadToCPUList(ret->tid.native_handle(), cpus); + } + }); + } + else { + ret->tid = thread(responderThread, ret); + if (!cpus.empty()) { + mapThreadToCPUList(ret->tid.native_handle(), cpus); + } + } + } - auto states = g_dstates.getCopy(); - states.push_back(ret); - std::stable_sort(states.begin(), states.end(), [](const decltype(ret)& a, const decltype(ret)& b) { - return a->order < b->order; - }); - g_dstates.setState(states); - return ret; - } ); + auto states = g_dstates.getCopy(); + states.push_back(ret); + std::stable_sort(states.begin(), states.end(), [](const decltype(ret)& a, const decltype(ret)& b) { + return a->order < b->order; + }); + g_dstates.setState(states); + return ret; + }); luaCtx.writeFunction("rmServer", - [](boost::variant, int, std::string> var) - { - setLuaSideEffect(); - shared_ptr server = nullptr; - auto states = g_dstates.getCopy(); - if (auto* rem = boost::get>(&var)) { - server = *rem; - } - else if (auto str = boost::get(&var)) { - const auto uuid = getUniqueID(*str); - for (auto& state : states) { - if (state->id == uuid) { - server = state; - } - } - } - else { - int idx = boost::get(var); - server = states.at(idx); - } - if (!server) { - throw std::runtime_error("unable to locate the requested server"); - } - auto localPools = g_pools.getCopy(); - for (const string& poolName : server->pools) { - removeServerFromPool(localPools, poolName, server); - } - /* the server might also be in the default pool */ - removeServerFromPool(localPools, "", server); - g_pools.setState(localPools); - states.erase(remove(states.begin(), states.end(), server), states.end()); - g_dstates.setState(states); - server->stop(); - } ); + [](boost::variant, int, std::string> var) { + setLuaSideEffect(); + shared_ptr server = nullptr; + auto states = g_dstates.getCopy(); + if (auto* rem = boost::get>(&var)) { + server = *rem; + } + else if (auto str = boost::get(&var)) { + const auto uuid = getUniqueID(*str); + for (auto& state : states) { + if (state->id == uuid) { + server = state; + } + } + } + else { + int idx = boost::get(var); + server = states.at(idx); + } + if (!server) { + throw std::runtime_error("unable to locate the requested server"); + } + auto localPools = g_pools.getCopy(); + for (const string& poolName : server->pools) { + removeServerFromPool(localPools, poolName, server); + } + /* the server might also be in the default pool */ + removeServerFromPool(localPools, "", server); + g_pools.setState(localPools); + states.erase(remove(states.begin(), states.end(), server), states.end()); + g_dstates.setState(states); + server->stop(); + }); luaCtx.writeFunction("truncateTC", [](bool tc) { setLuaSideEffect(); g_truncateTC=tc; }); luaCtx.writeFunction("fixupCase", [](bool fu) { setLuaSideEffect(); g_fixupCase=fu; }); luaCtx.writeFunction("addACL", [](const std::string& domain) { - setLuaSideEffect(); - g_ACL.modify([domain](NetmaskGroup& nmg) { nmg.addMask(domain); }); - }); + setLuaSideEffect(); + g_ACL.modify([domain](NetmaskGroup& nmg) { nmg.addMask(domain); }); + }); luaCtx.writeFunction("rmACL", [](const std::string& netmask) { setLuaSideEffect(); @@ -630,146 +671,146 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) }); luaCtx.writeFunction("setLocal", [client](const std::string& addr, boost::optional vars) { - setLuaSideEffect(); - if(client) - return; - if (g_configurationDone) { - g_outputBuffer="setLocal cannot be used at runtime!\n"; - return; - } - bool reusePort = false; - int tcpFastOpenQueueSize = 0; - int tcpListenQueueSize = 0; - size_t maxInFlightQueriesPerConn = 0; - size_t tcpMaxConcurrentConnections = 0; - std::string interface; - std::set cpus; - - parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConnections); + setLuaSideEffect(); + if (client) + return; + if (g_configurationDone) { + g_outputBuffer = "setLocal cannot be used at runtime!\n"; + return; + } + bool reusePort = false; + int tcpFastOpenQueueSize = 0; + int tcpListenQueueSize = 0; + size_t maxInFlightQueriesPerConn = 0; + size_t tcpMaxConcurrentConnections = 0; + std::string interface; + std::set cpus; - try { - ComboAddress loc(addr, 53); - for (auto it = g_frontends.begin(); it != g_frontends.end(); ) { - /* DoH, DoT and DNSCrypt frontends are separate */ - if ((*it)->tlsFrontend == nullptr && (*it)->dnscryptCtx == nullptr && (*it)->dohFrontend == nullptr) { - it = g_frontends.erase(it); - } - else { - ++it; - } - } + parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConnections); - // only works pre-startup, so no sync necessary - g_frontends.push_back(std::unique_ptr(new ClientState(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus))); - auto tcpCS = std::unique_ptr(new ClientState(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus)); - if (tcpListenQueueSize > 0) { - tcpCS->tcpListenQueueSize = tcpListenQueueSize; - } - if (maxInFlightQueriesPerConn > 0) { - tcpCS->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; + try { + ComboAddress loc(addr, 53); + for (auto it = g_frontends.begin(); it != g_frontends.end();) { + /* DoH, DoT and DNSCrypt frontends are separate */ + if ((*it)->tlsFrontend == nullptr && (*it)->dnscryptCtx == nullptr && (*it)->dohFrontend == nullptr) { + it = g_frontends.erase(it); } - if (tcpMaxConcurrentConnections > 0) { - tcpCS->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; + else { + ++it; } + } - g_frontends.push_back(std::move(tcpCS)); + // only works pre-startup, so no sync necessary + g_frontends.push_back(std::make_unique(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus)); + auto tcpCS = std::make_unique(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus); + if (tcpListenQueueSize > 0) { + tcpCS->tcpListenQueueSize = tcpListenQueueSize; } - catch(const std::exception& e) { - g_outputBuffer="Error: "+string(e.what())+"\n"; + if (maxInFlightQueriesPerConn > 0) { + tcpCS->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; } - }); + if (tcpMaxConcurrentConnections > 0) { + tcpCS->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; + } + + g_frontends.push_back(std::move(tcpCS)); + } + catch (const std::exception& e) { + g_outputBuffer = "Error: " + string(e.what()) + "\n"; + } + }); luaCtx.writeFunction("addLocal", [client](const std::string& addr, boost::optional vars) { - setLuaSideEffect(); - if(client) - return; - if (g_configurationDone) { - g_outputBuffer="addLocal cannot be used at runtime!\n"; - return; - } - bool reusePort = false; - int tcpFastOpenQueueSize = 0; - int tcpListenQueueSize = 0; - size_t maxInFlightQueriesPerConn = 0; - size_t tcpMaxConcurrentConnections = 0; - std::string interface; - std::set cpus; + setLuaSideEffect(); + if (client) + return; + if (g_configurationDone) { + g_outputBuffer = "addLocal cannot be used at runtime!\n"; + return; + } + bool reusePort = false; + int tcpFastOpenQueueSize = 0; + int tcpListenQueueSize = 0; + size_t maxInFlightQueriesPerConn = 0; + size_t tcpMaxConcurrentConnections = 0; + std::string interface; + std::set cpus; - parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConnections); + parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConnections); - try { - ComboAddress loc(addr, 53); - // only works pre-startup, so no sync necessary - g_frontends.push_back(std::unique_ptr(new ClientState(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus))); - auto tcpCS = std::unique_ptr(new ClientState(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus)); - if (tcpListenQueueSize > 0) { - tcpCS->tcpListenQueueSize = tcpListenQueueSize; - } - if (maxInFlightQueriesPerConn > 0) { - tcpCS->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; - } - if (tcpMaxConcurrentConnections > 0) { - tcpCS->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; - } - g_frontends.push_back(std::move(tcpCS)); + try { + ComboAddress loc(addr, 53); + // only works pre-startup, so no sync necessary + g_frontends.push_back(std::make_unique(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus)); + auto tcpCS = std::make_unique(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus); + if (tcpListenQueueSize > 0) { + tcpCS->tcpListenQueueSize = tcpListenQueueSize; } - catch(std::exception& e) { - g_outputBuffer="Error: "+string(e.what())+"\n"; - errlog("Error while trying to listen on %s: %s\n", addr, string(e.what())); + if (maxInFlightQueriesPerConn > 0) { + tcpCS->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; } - }); - - luaCtx.writeFunction("setACL", [](boost::variant>> inp) { - setLuaSideEffect(); - NetmaskGroup nmg; - if(auto str = boost::get(&inp)) { - nmg.addMask(*str); + if (tcpMaxConcurrentConnections > 0) { + tcpCS->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; } - else for(const auto& p : boost::get>>(inp)) { - nmg.addMask(p.second); + g_frontends.push_back(std::move(tcpCS)); + } + catch (std::exception& e) { + g_outputBuffer = "Error: " + string(e.what()) + "\n"; + errlog("Error while trying to listen on %s: %s\n", addr, string(e.what())); + } + }); + + luaCtx.writeFunction("setACL", [](boost::variant>> inp) { + setLuaSideEffect(); + NetmaskGroup nmg; + if (auto str = boost::get(&inp)) { + nmg.addMask(*str); + } + else + for (const auto& p : boost::get>>(inp)) { + nmg.addMask(p.second); } - g_ACL.setState(nmg); + g_ACL.setState(nmg); }); luaCtx.writeFunction("setACLFromFile", [](const std::string& file) { - setLuaSideEffect(); - NetmaskGroup nmg; - - ifstream ifs(file); - if(!ifs) { - throw std::runtime_error("Could not open '"+file+"': "+stringerror()); - } - - string::size_type pos; - string line; - while(getline(ifs,line)) { - pos=line.find('#'); - if(pos!=string::npos) - line.resize(pos); - boost::trim(line); - if(line.empty()) - continue; + setLuaSideEffect(); + NetmaskGroup nmg; - nmg.addMask(line); - } + ifstream ifs(file); + if (!ifs) { + throw std::runtime_error("Could not open '" + file + "': " + stringerror()); + } + + string::size_type pos; + string line; + while (getline(ifs, line)) { + pos = line.find('#'); + if (pos != string::npos) + line.resize(pos); + boost::trim(line); + if (line.empty()) + continue; + + nmg.addMask(line); + } - g_ACL.setState(nmg); + g_ACL.setState(nmg); }); luaCtx.writeFunction("showACL", []() { - setLuaNoSideEffect(); - vector vec; - - g_ACL.getLocal()->toStringVector(&vec); + setLuaNoSideEffect(); + vector vec; - for(const auto& s : vec) - g_outputBuffer+=s+"\n"; + g_ACL.getLocal()->toStringVector(&vec); - }); + for (const auto& s : vec) + g_outputBuffer += s + "\n"; + }); luaCtx.writeFunction("shutdown", []() { #ifdef HAVE_SYSTEMD - sd_notify(0, "STOPPING=1"); + sd_notify(0, "STOPPING=1"); #endif /* HAVE_SYSTEMD */ #if 0 // Useful for debugging leaks, but might lead to race under load @@ -778,468 +819,510 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) frontend->cleanup(); } g_tlslocals.clear(); + g_rings.clear(); #endif /* 0 */ - _exit(0); - } ); + _exit(0); + }); - typedef std::unordered_map > showserversopts_t; + typedef std::unordered_map> showserversopts_t; luaCtx.writeFunction("showServers", [](boost::optional vars) { - setLuaNoSideEffect(); - bool showUUIDs = false; - if (vars) { - if (vars->count("showUUIDs")) { - showUUIDs = boost::get((*vars)["showUUIDs"]); - } + setLuaNoSideEffect(); + bool showUUIDs = false; + if (vars) { + if (vars->count("showUUIDs")) { + showUUIDs = boost::get((*vars)["showUUIDs"]); + } + } + try { + ostringstream ret; + boost::format fmt; + if (showUUIDs) { + fmt = boost::format("%1$-3d %15$-36s %2$-20.20s %|62t|%3% %|92t|%4$5s %|88t|%5$7.1f %|103t|%6$7d %|106t|%7$3d %|115t|%8$2d %|117t|%9$10d %|123t|%10$7d %|128t|%11$5.1f %|146t|%12$5.1f %|152t|%13$11d %14%"); + // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 + ret << (fmt % "#" % "Name" % "Address" % "State" % "Qps" % "Qlim" % "Ord" % "Wt" % "Queries" % "Drops" % "Drate" % "Lat" % "Outstanding" % "Pools" % "UUID") << endl; + } + else { + fmt = boost::format("%1$-3d %2$-20.20s %|25t|%3% %|55t|%4$5s %|51t|%5$7.1f %|66t|%6$7d %|69t|%7$3d %|78t|%8$2d %|80t|%9$10d %|86t|%10$7d %|91t|%11$5.1f %|109t|%12$5.1f %|115t|%13$11d %14%"); + ret << (fmt % "#" % "Name" % "Address" % "State" % "Qps" % "Qlim" % "Ord" % "Wt" % "Queries" % "Drops" % "Drate" % "Lat" % "Outstanding" % "Pools") << endl; } - try { - ostringstream ret; - boost::format fmt; - if (showUUIDs) { - fmt = boost::format("%1$-3d %15$-36s %2$-20.20s %|62t|%3% %|92t|%4$5s %|88t|%5$7.1f %|103t|%6$7d %|106t|%7$3d %|115t|%8$2d %|117t|%9$10d %|123t|%10$7d %|128t|%11$5.1f %|146t|%12$5.1f %|152t|%13$11d %14%" ); - // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - ret << (fmt % "#" % "Name" % "Address" % "State" % "Qps" % "Qlim" % "Ord" % "Wt" % "Queries" % "Drops" % "Drate" % "Lat" % "Outstanding" % "Pools" % "UUID") << endl; - } else { - fmt = boost::format("%1$-3d %2$-20.20s %|25t|%3% %|55t|%4$5s %|51t|%5$7.1f %|66t|%6$7d %|69t|%7$3d %|78t|%8$2d %|80t|%9$10d %|86t|%10$7d %|91t|%11$5.1f %|109t|%12$5.1f %|115t|%13$11d %14%" ); - ret << (fmt % "#" % "Name" % "Address" % "State" % "Qps" % "Qlim" % "Ord" % "Wt" % "Queries" % "Drops" % "Drate" % "Lat" % "Outstanding" % "Pools") << endl; - } - uint64_t totQPS{0}, totQueries{0}, totDrops{0}; - int counter=0; - auto states = g_dstates.getLocal(); - for(const auto& s : *states) { - string status = s->getStatus(); - string pools; - for(auto& p : s->pools) { - if(!pools.empty()) - pools+=" "; - pools+=p; - } - if (showUUIDs) { - ret << (fmt % counter % s->getName() % s->remote.toStringWithPort() % - status % - s->queryLoad % s->qps.getRate() % s->order % s->weight % s->queries.load() % s->reuseds.load() % (s->dropRate) % (s->latencyUsec/1000.0) % s->outstanding.load() % pools % s->id) << endl; - } else { - ret << (fmt % counter % s->getName() % s->remote.toStringWithPort() % - status % - s->queryLoad % s->qps.getRate() % s->order % s->weight % s->queries.load() % s->reuseds.load() % (s->dropRate) % (s->latencyUsec/1000.0) % s->outstanding.load() % pools) << endl; - } - totQPS += s->queryLoad; - totQueries += s->queries.load(); - totDrops += s->reuseds.load(); - ++counter; + uint64_t totQPS{0}, totQueries{0}, totDrops{0}; + int counter = 0; + auto states = g_dstates.getLocal(); + for (const auto& s : *states) { + string status = s->getStatus(); + string pools; + for (auto& p : s->pools) { + if (!pools.empty()) + pools += " "; + pools += p; } if (showUUIDs) { - ret<< (fmt % "All" % "" % "" % "" - % - (double)totQPS % "" % "" % "" % totQueries % totDrops % "" % "" % "" % "" % "" ) << endl; - } else { - ret<< (fmt % "All" % "" % "" % "" - % - (double)totQPS % "" % "" % "" % totQueries % totDrops % "" % "" % "" % "" ) << endl; + ret << (fmt % counter % s->getName() % s->remote.toStringWithPort() % status % s->queryLoad % s->qps.getRate() % s->order % s->weight % s->queries.load() % s->reuseds.load() % (s->dropRate) % (s->latencyUsec / 1000.0) % s->outstanding.load() % pools % s->id) << endl; } - - g_outputBuffer=ret.str(); - } catch(std::exception& e) { - g_outputBuffer=e.what(); - throw; + else { + ret << (fmt % counter % s->getName() % s->remote.toStringWithPort() % status % s->queryLoad % s->qps.getRate() % s->order % s->weight % s->queries.load() % s->reuseds.load() % (s->dropRate) % (s->latencyUsec / 1000.0) % s->outstanding.load() % pools) << endl; + } + totQPS += s->queryLoad; + totQueries += s->queries.load(); + totDrops += s->reuseds.load(); + ++counter; } - }); + if (showUUIDs) { + ret << (fmt % "All" % "" % "" % "" + % (double)totQPS % "" % "" % "" % totQueries % totDrops % "" % "" % "" % "" % "") + << endl; + } + else { + ret << (fmt % "All" % "" % "" % "" + % (double)totQPS % "" % "" % "" % totQueries % totDrops % "" % "" % "" % "") + << endl; + } + + g_outputBuffer = ret.str(); + } + catch (std::exception& e) { + g_outputBuffer = e.what(); + throw; + } + }); luaCtx.writeFunction("getServers", []() { - setLuaNoSideEffect(); - vector > > ret; - int count=1; - for(const auto& s : g_dstates.getCopy()) { - ret.push_back(make_pair(count++, s)); - } - return ret; - }); + setLuaNoSideEffect(); + vector>> ret; + int count = 1; + for (const auto& s : g_dstates.getCopy()) { + ret.push_back(make_pair(count++, s)); + } + return ret; + }); luaCtx.writeFunction("getPoolServers", [](string pool) { - const auto poolServers = getDownstreamCandidates(g_pools.getCopy(), pool); - return *poolServers; - }); + const auto poolServers = getDownstreamCandidates(g_pools.getCopy(), pool); + return *poolServers; + }); luaCtx.writeFunction("getServer", [client](boost::variant i) { - if (client) { - return std::make_shared(ComboAddress(), ComboAddress(), 0, std::string(), 1, false); - } - auto states = g_dstates.getCopy(); - if (auto str = boost::get(&i)) { - const auto uuid = getUniqueID(*str); - for (auto& state : states) { - if (state->id == uuid) { - return state; - } + if (client) { + return std::make_shared(ComboAddress()); + } + auto states = g_dstates.getCopy(); + if (auto str = boost::get(&i)) { + const auto uuid = getUniqueID(*str); + for (auto& state : states) { + if (state->id == uuid) { + return state; } } - else if (auto pos = boost::get(&i)) { - return states.at(*pos); - } + } + else if (auto pos = boost::get(&i)) { + return states.at(*pos); + } - g_outputBuffer = "Error: no rule matched\n"; - return std::shared_ptr(nullptr); - }); + g_outputBuffer = "Error: no rule matched\n"; + return std::shared_ptr(nullptr); + }); - luaCtx.writeFunction("carbonServer", [](const std::string& address, boost::optional ourName, - boost::optional interval, boost::optional namespace_name, - boost::optional instance_name) { - setLuaSideEffect(); - auto ours = g_carbon.getCopy(); - ours.push_back({ - ComboAddress(address, 2003), - (namespace_name && !namespace_name->empty()) ? *namespace_name : "dnsdist", - ourName ? *ourName : "", - (instance_name && !instance_name->empty()) ? *instance_name : "main" , - interval ? *interval : 30 - }); - g_carbon.setState(ours); - }); - - luaCtx.writeFunction("webserver", [client,configCheck](const std::string& address, const boost::optional password, const boost::optional apiKey, const boost::optional > customHeaders, const boost::optional acl) { - setLuaSideEffect(); - ComboAddress local; - try { - local = ComboAddress(address); - } - catch (const PDNSException& e) { - throw std::runtime_error(std::string("Error parsing the bind address for the webserver: ") + e.reason); - } + luaCtx.writeFunction("carbonServer", [](const std::string& address, boost::optional ourName, boost::optional interval, boost::optional namespace_name, boost::optional instance_name) { + setLuaSideEffect(); + auto ours = g_carbon.getCopy(); + ours.push_back({ComboAddress(address, 2003), + (namespace_name && !namespace_name->empty()) ? *namespace_name : "dnsdist", + ourName ? *ourName : "", + (instance_name && !instance_name->empty()) ? *instance_name : "main", + interval ? *interval : 30}); + g_carbon.setState(ours); + }); - if (client || configCheck) { - return; - } + luaCtx.writeFunction("webserver", [client, configCheck](const std::string& address, boost::optional password, boost::optional apiKey, const boost::optional> customHeaders, const boost::optional acl) { + setLuaSideEffect(); + ComboAddress local; + try { + local = ComboAddress(address); + } + catch (const PDNSException& e) { + throw std::runtime_error(std::string("Error parsing the bind address for the webserver: ") + e.reason); + } - if (password || apiKey || customHeaders || acl) { - warnlog("Passing additional parameters to 'webserver()' is deprecated, please use 'setWebserverConfig()' instead."); - } + if (client || configCheck) { + return; + } - try { - int sock = SSocket(local.sin4.sin_family, SOCK_STREAM, 0); - SSetsockopt(sock, SOL_SOCKET, SO_REUSEADDR, 1); - SBind(sock, local); - SListen(sock, 5); - auto launch=[sock, local, password, apiKey, customHeaders, acl]() { - if (password) { - setWebserverPassword(*password); - } + if (password || apiKey || customHeaders || acl) { + warnlog("Passing additional parameters to 'webserver()' is deprecated, please use 'setWebserverConfig()' instead."); + } - if (apiKey) { - setWebserverAPIKey(apiKey); + try { + int sock = SSocket(local.sin4.sin_family, SOCK_STREAM, 0); + SSetsockopt(sock, SOL_SOCKET, SO_REUSEADDR, 1); + SBind(sock, local); + SListen(sock, 5); + auto launch = [sock, local, password, apiKey, customHeaders, acl]() { + if (password) { + auto holder = make_unique(std::string(*password), false); + if (!holder->wasHashed() && holder->isHashingAvailable()) { + infolog("Passing a plain-text password to 'webserver()' is deprecated, please use 'setWebserverConfig()' instead and consider generating a hashed password using 'hashPassword()'."); } - if (customHeaders) { - setWebserverCustomHeaders(customHeaders); - } + setWebserverPassword(std::move(holder)); + } - if (acl) { - setWebserverACL(*acl); - } - thread t(dnsdistWebserverThread, sock, local); - t.detach(); - }; - if (g_launchWork) { - g_launchWork->push_back(launch); + if (apiKey) { + auto holder = make_unique(std::string(*apiKey), false); + setWebserverAPIKey(std::move(holder)); } - else { - launch(); + + if (customHeaders) { + setWebserverCustomHeaders(customHeaders); + } + + if (acl) { + setWebserverACL(*acl); } + thread t(dnsdistWebserverThread, sock, local); + t.detach(); + }; + if (g_launchWork) { + g_launchWork->push_back(launch); } - catch (const std::exception& e) { - g_outputBuffer="Unable to bind to webserver socket on " + local.toStringWithPort() + ": " + e.what(); - errlog("Unable to bind to webserver socket on %s: %s", local.toStringWithPort(), e.what()); + else { + launch(); } + } + catch (const std::exception& e) { + g_outputBuffer = "Unable to bind to webserver socket on " + local.toStringWithPort() + ": " + e.what(); + errlog("Unable to bind to webserver socket on %s: %s", local.toStringWithPort(), e.what()); + } + }); - }); - - typedef std::unordered_map> > webserveropts_t; + typedef std::unordered_map>> webserveropts_t; luaCtx.writeFunction("setWebserverConfig", [](boost::optional vars) { - setLuaSideEffect(); + setLuaSideEffect(); - if (!vars) { - return ; - } + if (!vars) { + return; + } - if (vars->count("password")) { - const std::string password = boost::get(vars->at("password")); + bool hashPlaintextCredentials = false; + if (vars->count("hashPlaintextCredentials")) { + hashPlaintextCredentials = boost::get(vars->at("hashPlaintextCredentials")); + } - setWebserverPassword(password); + if (vars->count("password")) { + std::string password = boost::get(vars->at("password")); + auto holder = make_unique(std::move(password), hashPlaintextCredentials); + if (!holder->wasHashed() && holder->isHashingAvailable()) { + infolog("Passing a plain-text password via the 'password' parameter to 'setWebserverConfig()' is not advised, please consider generating a hashed one using 'hashPassword()' instead."); } - if (vars->count("apiKey")) { - const std::string apiKey = boost::get(vars->at("apiKey")); + setWebserverPassword(std::move(holder)); + } - setWebserverAPIKey(apiKey); + if (vars->count("apiKey")) { + std::string apiKey = boost::get(vars->at("apiKey")); + auto holder = make_unique(std::move(apiKey), hashPlaintextCredentials); + if (!holder->wasHashed() && holder->isHashingAvailable()) { + infolog("Passing a plain-text API key via the 'apiKey' parameter to 'setWebserverConfig()' is not advised, please consider generating a hashed one using 'hashPassword()' instead."); } - if (vars->count("acl")) { - const std::string acl = boost::get(vars->at("acl")); - - setWebserverACL(acl); - } + setWebserverAPIKey(std::move(holder)); + } - if (vars->count("customHeaders")) { - const boost::optional > headers = boost::get >(vars->at("customHeaders")); + if (vars->count("acl")) { + const std::string acl = boost::get(vars->at("acl")); - setWebserverCustomHeaders(headers); - } + setWebserverACL(acl); + } - if (vars->count("statsRequireAuthentication")) { - setWebserverStatsRequireAuthentication(boost::get(vars->at("statsRequireAuthentication"))); - } + if (vars->count("customHeaders")) { + const boost::optional> headers = boost::get>(vars->at("customHeaders")); - if (vars->count("maxConcurrentConnections")) { - setWebserverMaxConcurrentConnections(std::stoi(boost::get(vars->at("maxConcurrentConnections")))); - } - }); + setWebserverCustomHeaders(headers); + } - luaCtx.writeFunction("controlSocket", [client,configCheck](const std::string& str) { - setLuaSideEffect(); - ComboAddress local(str, 5199); + if (vars->count("statsRequireAuthentication")) { + setWebserverStatsRequireAuthentication(boost::get(vars->at("statsRequireAuthentication"))); + } - if(client || configCheck) { - g_serverControl = local; - return; - } + if (vars->count("maxConcurrentConnections")) { + setWebserverMaxConcurrentConnections(std::stoi(boost::get(vars->at("maxConcurrentConnections")))); + } + }); - g_consoleEnabled = true; + luaCtx.writeFunction("hashPassword", [](const std::string& password, boost::optional workFactor) { + if (workFactor) { + return hashPassword(password, *workFactor, CredentialsHolder::s_defaultParallelFactor, CredentialsHolder::s_defaultBlockSize); + } + return hashPassword(password); + }); + + luaCtx.writeFunction("controlSocket", [client, configCheck](const std::string& str) { + setLuaSideEffect(); + ComboAddress local(str, 5199); + + if (client || configCheck) { + g_serverControl = local; + return; + } + + g_consoleEnabled = true; #ifdef HAVE_LIBSODIUM - if (g_configurationDone && g_consoleKey.empty()) { - warnlog("Warning, the console has been enabled via 'controlSocket()' but no key has been set with 'setKey()' so all connections will fail until a key has been set"); - } + if (g_configurationDone && g_consoleKey.empty()) { + warnlog("Warning, the console has been enabled via 'controlSocket()' but no key has been set with 'setKey()' so all connections will fail until a key has been set"); + } #endif - try { - int sock = SSocket(local.sin4.sin_family, SOCK_STREAM, 0); - SSetsockopt(sock, SOL_SOCKET, SO_REUSEADDR, 1); - SBind(sock, local); - SListen(sock, 5); - auto launch=[sock, local]() { - thread t(controlThread, sock, local); - t.detach(); - }; - if (g_launchWork) { - g_launchWork->push_back(launch); - } - else { - launch(); - } + try { + int sock = SSocket(local.sin4.sin_family, SOCK_STREAM, 0); + SSetsockopt(sock, SOL_SOCKET, SO_REUSEADDR, 1); + SBind(sock, local); + SListen(sock, 5); + auto launch = [sock, local]() { + thread t(controlThread, sock, local); + t.detach(); + }; + if (g_launchWork) { + g_launchWork->push_back(launch); } - catch(std::exception& e) { - g_outputBuffer="Unable to bind to control socket on " + local.toStringWithPort() + ": " + e.what(); - errlog("Unable to bind to control socket on %s: %s", local.toStringWithPort(), e.what()); + else { + launch(); } - }); + } + catch (std::exception& e) { + g_outputBuffer = "Unable to bind to control socket on " + local.toStringWithPort() + ": " + e.what(); + errlog("Unable to bind to control socket on %s: %s", local.toStringWithPort(), e.what()); + } + }); luaCtx.writeFunction("addConsoleACL", [](const std::string& netmask) { - setLuaSideEffect(); + setLuaSideEffect(); #ifndef HAVE_LIBSODIUM - warnlog("Allowing remote access to the console while libsodium support has not been enabled is not secure, and will result in cleartext communications"); + warnlog("Allowing remote access to the console while libsodium support has not been enabled is not secure, and will result in cleartext communications"); #endif - g_consoleACL.modify([netmask](NetmaskGroup& nmg) { nmg.addMask(netmask); }); - }); + g_consoleACL.modify([netmask](NetmaskGroup& nmg) { nmg.addMask(netmask); }); + }); - luaCtx.writeFunction("setConsoleACL", [](boost::variant>> inp) { - setLuaSideEffect(); + luaCtx.writeFunction("setConsoleACL", [](boost::variant>> inp) { + setLuaSideEffect(); #ifndef HAVE_LIBSODIUM - warnlog("Allowing remote access to the console while libsodium support has not been enabled is not secure, and will result in cleartext communications"); + warnlog("Allowing remote access to the console while libsodium support has not been enabled is not secure, and will result in cleartext communications"); #endif - NetmaskGroup nmg; - if(auto str = boost::get(&inp)) { - nmg.addMask(*str); - } - else for(const auto& p : boost::get>>(inp)) { - nmg.addMask(p.second); + NetmaskGroup nmg; + if (auto str = boost::get(&inp)) { + nmg.addMask(*str); + } + else + for (const auto& p : boost::get>>(inp)) { + nmg.addMask(p.second); } - g_consoleACL.setState(nmg); + g_consoleACL.setState(nmg); }); luaCtx.writeFunction("showConsoleACL", []() { - setLuaNoSideEffect(); + setLuaNoSideEffect(); #ifndef HAVE_LIBSODIUM - warnlog("Allowing remote access to the console while libsodium support has not been enabled is not secure, and will result in cleartext communications"); + warnlog("Allowing remote access to the console while libsodium support has not been enabled is not secure, and will result in cleartext communications"); #endif - vector vec; - g_consoleACL.getLocal()->toStringVector(&vec); + vector vec; + g_consoleACL.getLocal()->toStringVector(&vec); - for(const auto& s : vec) { - g_outputBuffer += s + "\n"; - } - }); + for (const auto& s : vec) { + g_outputBuffer += s + "\n"; + } + }); luaCtx.writeFunction("setConsoleMaximumConcurrentConnections", [](size_t max) { - setLuaSideEffect(); - setConsoleMaximumConcurrentConnections(max); + setLuaSideEffect(); + setConsoleMaximumConcurrentConnections(max); }); luaCtx.writeFunction("clearQueryCounters", []() { - unsigned int size{0}; - { - auto records = g_qcount.records.write_lock(); - size = records->size(); - records->clear(); - } + unsigned int size{0}; + { + auto records = g_qcount.records.write_lock(); + size = records->size(); + records->clear(); + } - boost::format fmt("%d records cleared from query counter buffer\n"); - g_outputBuffer = (fmt % size).str(); - }); + boost::format fmt("%d records cleared from query counter buffer\n"); + g_outputBuffer = (fmt % size).str(); + }); luaCtx.writeFunction("getQueryCounters", [](boost::optional optMax) { - setLuaNoSideEffect(); - auto records = g_qcount.records.read_lock(); - g_outputBuffer = "query counting is currently: "; - g_outputBuffer+= g_qcount.enabled ? "enabled" : "disabled"; - g_outputBuffer+= (boost::format(" (%d records in buffer)\n") % records->size()).str(); - - boost::format fmt("%-3d %s: %d request(s)\n"); - unsigned int max = optMax ? *optMax : 10; - unsigned int index{1}; - for (auto it = records->begin(); it != records->end() && index <= max; ++it, ++index) { - g_outputBuffer += (fmt % index % it->first % it->second).str(); - } - }); + setLuaNoSideEffect(); + auto records = g_qcount.records.read_lock(); + g_outputBuffer = "query counting is currently: "; + g_outputBuffer += g_qcount.enabled ? "enabled" : "disabled"; + g_outputBuffer += (boost::format(" (%d records in buffer)\n") % records->size()).str(); + + boost::format fmt("%-3d %s: %d request(s)\n"); + unsigned int max = optMax ? *optMax : 10; + unsigned int index{1}; + for (auto it = records->begin(); it != records->end() && index <= max; ++it, ++index) { + g_outputBuffer += (fmt % index % it->first % it->second).str(); + } + }); - luaCtx.writeFunction("setQueryCount", [](bool enabled) { g_qcount.enabled=enabled; }); + luaCtx.writeFunction("setQueryCount", [](bool enabled) { g_qcount.enabled = enabled; }); luaCtx.writeFunction("setQueryCountFilter", [](QueryCountFilter func) { - g_qcount.filter = func; - }); + g_qcount.filter = func; + }); luaCtx.writeFunction("makeKey", []() { - setLuaNoSideEffect(); - g_outputBuffer="setKey("+newKey()+")\n"; - }); + setLuaNoSideEffect(); + g_outputBuffer = "setKey(" + newKey() + ")\n"; + }); luaCtx.writeFunction("setKey", [](const std::string& key) { - if(!g_configurationDone && ! g_consoleKey.empty()) { // this makes sure the commandline -k key prevails over dnsdist.conf - return; // but later setKeys() trump the -k value again - } + if (!g_configurationDone && !g_consoleKey.empty()) { // this makes sure the commandline -k key prevails over dnsdist.conf + return; // but later setKeys() trump the -k value again + } #ifndef HAVE_LIBSODIUM - warnlog("Calling setKey() while libsodium support has not been enabled is not secure, and will result in cleartext communications"); + warnlog("Calling setKey() while libsodium support has not been enabled is not secure, and will result in cleartext communications"); #endif - setLuaSideEffect(); - string newkey; - if(B64Decode(key, newkey) < 0) { - g_outputBuffer=string("Unable to decode ")+key+" as Base64"; - errlog("%s", g_outputBuffer); - } - else - g_consoleKey=newkey; - }); + setLuaSideEffect(); + string newkey; + if (B64Decode(key, newkey) < 0) { + g_outputBuffer = string("Unable to decode ") + key + " as Base64"; + errlog("%s", g_outputBuffer); + } + else + g_consoleKey = newkey; + }); luaCtx.writeFunction("clearConsoleHistory", []() { - clearConsoleHistory(); - }); + clearConsoleHistory(); + }); - luaCtx.writeFunction("testCrypto", [](boost::optional optTestMsg) - { - setLuaNoSideEffect(); + luaCtx.writeFunction("testCrypto", [](boost::optional optTestMsg) { + setLuaNoSideEffect(); #ifdef HAVE_LIBSODIUM - try { - string testmsg; - - if (optTestMsg) { - testmsg = *optTestMsg; - } - else { - testmsg = "testStringForCryptoTests"; - } - - SodiumNonce sn, sn2; - sn.init(); - sn2=sn; - string encrypted = sodEncryptSym(testmsg, g_consoleKey, sn); - string decrypted = sodDecryptSym(encrypted, g_consoleKey, sn2); - - sn.increment(); - sn2.increment(); - - encrypted = sodEncryptSym(testmsg, g_consoleKey, sn); - decrypted = sodDecryptSym(encrypted, g_consoleKey, sn2); - - if(testmsg == decrypted) - g_outputBuffer="Everything is ok!\n"; - else - g_outputBuffer="Crypto failed.. (the decoded value does not match the cleartext one)\n"; - } - catch(const std::exception& e) { - g_outputBuffer="Crypto failed: "+std::string(e.what())+"\n"; - } - catch(...) { - g_outputBuffer="Crypto failed..\n"; - } + try { + string testmsg; + + if (optTestMsg) { + testmsg = *optTestMsg; + } + else { + testmsg = "testStringForCryptoTests"; + } + + SodiumNonce sn, sn2; + sn.init(); + sn2 = sn; + string encrypted = sodEncryptSym(testmsg, g_consoleKey, sn); + string decrypted = sodDecryptSym(encrypted, g_consoleKey, sn2); + + sn.increment(); + sn2.increment(); + + encrypted = sodEncryptSym(testmsg, g_consoleKey, sn); + decrypted = sodDecryptSym(encrypted, g_consoleKey, sn2); + + if (testmsg == decrypted) + g_outputBuffer = "Everything is ok!\n"; + else + g_outputBuffer = "Crypto failed.. (the decoded value does not match the cleartext one)\n"; + } + catch (const std::exception& e) { + g_outputBuffer = "Crypto failed: " + std::string(e.what()) + "\n"; + } + catch (...) { + g_outputBuffer = "Crypto failed..\n"; + } #else - g_outputBuffer="Crypto not available.\n"; + g_outputBuffer = "Crypto not available.\n"; #endif - }); + }); - luaCtx.writeFunction("setTCPRecvTimeout", [](int timeout) { g_tcpRecvTimeout=timeout; }); + luaCtx.writeFunction("setTCPRecvTimeout", [](int timeout) { g_tcpRecvTimeout = timeout; }); - luaCtx.writeFunction("setTCPSendTimeout", [](int timeout) { g_tcpSendTimeout=timeout; }); + luaCtx.writeFunction("setTCPSendTimeout", [](int timeout) { g_tcpSendTimeout = timeout; }); - luaCtx.writeFunction("setUDPTimeout", [](int timeout) { g_udpTimeout=timeout; }); + luaCtx.writeFunction("setUDPTimeout", [](int timeout) { g_udpTimeout = timeout; }); - luaCtx.writeFunction("setMaxUDPOutstanding", [](uint16_t max) { - if (!g_configurationDone) { - g_maxOutstanding = max; - } else { - g_outputBuffer="Max UDP outstanding cannot be altered at runtime!\n"; - } - }); + luaCtx.writeFunction("setMaxUDPOutstanding", [](uint64_t max) { + if (!g_configurationDone) { + checkParameterBound("setMaxUDPOutstanding", max); + g_maxOutstanding = max; + } + else { + g_outputBuffer = "Max UDP outstanding cannot be altered at runtime!\n"; + } + }); luaCtx.writeFunction("setMaxTCPClientThreads", [](uint64_t max) { - if (!g_configurationDone) { - g_maxTCPClientThreads = max; - } else { - g_outputBuffer="Maximum TCP client threads count cannot be altered at runtime!\n"; - } - }); + if (!g_configurationDone) { + g_maxTCPClientThreads = max; + } + else { + g_outputBuffer = "Maximum TCP client threads count cannot be altered at runtime!\n"; + } + }); luaCtx.writeFunction("setMaxTCPQueuedConnections", [](uint64_t max) { - if (!g_configurationDone) { - g_maxTCPQueuedConnections = max; - } else { - g_outputBuffer="The maximum number of queued TCP connections cannot be altered at runtime!\n"; - } - }); + if (!g_configurationDone) { + g_maxTCPQueuedConnections = max; + } + else { + g_outputBuffer = "The maximum number of queued TCP connections cannot be altered at runtime!\n"; + } + }); luaCtx.writeFunction("setMaxTCPQueriesPerConnection", [](size_t max) { - if (!g_configurationDone) { - g_maxTCPQueriesPerConn = max; - } else { - g_outputBuffer="The maximum number of queries per TCP connection cannot be altered at runtime!\n"; - } - }); + if (!g_configurationDone) { + g_maxTCPQueriesPerConn = max; + } + else { + g_outputBuffer = "The maximum number of queries per TCP connection cannot be altered at runtime!\n"; + } + }); luaCtx.writeFunction("setMaxTCPConnectionsPerClient", [](size_t max) { - if (!g_configurationDone) { - g_maxTCPConnectionsPerClient = max; - } else { - g_outputBuffer="The maximum number of TCP connection per client cannot be altered at runtime!\n"; - } - }); + if (!g_configurationDone) { + g_maxTCPConnectionsPerClient = max; + } + else { + g_outputBuffer = "The maximum number of TCP connection per client cannot be altered at runtime!\n"; + } + }); luaCtx.writeFunction("setMaxTCPConnectionDuration", [](size_t max) { - if (!g_configurationDone) { - g_maxTCPConnectionDuration = max; - } else { - g_outputBuffer="The maximum duration of a TCP connection cannot be altered at runtime!\n"; - } - }); + if (!g_configurationDone) { + g_maxTCPConnectionDuration = max; + } + else { + g_outputBuffer = "The maximum duration of a TCP connection cannot be altered at runtime!\n"; + } + }); luaCtx.writeFunction("setMaxCachedTCPConnectionsPerDownstream", [](size_t max) { - setMaxCachedTCPConnectionsPerDownstream(max); - }); + DownstreamTCPConnectionsManager::setMaxIdleConnectionsPerDownstream(max); + }); - luaCtx.writeFunction("setOutgoingTLSSessionsCacheMaxTicketsPerBackend", [](uint16_t max) { + luaCtx.writeFunction("setMaxIdleDoHConnectionsPerDownstream", [](size_t max) { + setDoHDownstreamMaxIdleConnectionsPerBackend(max); + }); + + luaCtx.writeFunction("setOutgoingDoHWorkerThreads", [](uint64_t workers) { + if (!g_configurationDone) { + g_outgoingDoHWorkerThreads = workers; + } + else { + g_outputBuffer = "The amount of outgoing DoH worker threads cannot be altered at runtime!\n"; + } + }); + + luaCtx.writeFunction("setOutgoingTLSSessionsCacheMaxTicketsPerBackend", [](uint64_t max) { if (g_configurationDone) { g_outputBuffer = "setOutgoingTLSSessionsCacheMaxTicketsPerBackend() cannot be called at runtime!\n"; return; @@ -1263,762 +1346,815 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) TLSSessionCache::setSessionValidity(validity); }); - luaCtx.writeFunction("setCacheCleaningDelay", [](uint32_t delay) { g_cacheCleaningDelay = delay; }); + luaCtx.writeFunction("setCacheCleaningDelay", [](uint64_t delay) { + checkParameterBound("setCacheCleaningDelay", delay, std::numeric_limits::max()); + g_cacheCleaningDelay = delay; + }); luaCtx.writeFunction("setCacheCleaningPercentage", [](uint16_t percentage) { if (percentage < 100) g_cacheCleaningPercentage = percentage; else g_cacheCleaningPercentage = 100; }); - luaCtx.writeFunction("setECSSourcePrefixV4", [](uint16_t prefix) { g_ECSSourcePrefixV4=prefix; }); + luaCtx.writeFunction("setECSSourcePrefixV4", [](uint16_t prefix) { g_ECSSourcePrefixV4 = prefix; }); - luaCtx.writeFunction("setECSSourcePrefixV6", [](uint16_t prefix) { g_ECSSourcePrefixV6=prefix; }); + luaCtx.writeFunction("setECSSourcePrefixV6", [](uint16_t prefix) { g_ECSSourcePrefixV6 = prefix; }); - luaCtx.writeFunction("setECSOverride", [](bool override) { g_ECSOverride=override; }); + luaCtx.writeFunction("setECSOverride", [](bool override) { g_ECSOverride = override; }); luaCtx.writeFunction("showDynBlocks", []() { - setLuaNoSideEffect(); - auto slow = g_dynblockNMG.getCopy(); - struct timespec now; - gettime(&now); - boost::format fmt("%-24s %8d %8d %-10s %-20s %s\n"); - g_outputBuffer = (fmt % "What" % "Seconds" % "Blocks" % "Warning" % "Action" % "Reason").str(); - for(const auto& e: slow) { - if (now < e.second.until) { - uint64_t counter = e.second.blocks; - if (g_defaultBPFFilter && e.second.bpf) { - counter += g_defaultBPFFilter->getHits(e.first.getNetwork()); - } - g_outputBuffer+= (fmt % e.first.toString() % (e.second.until.tv_sec - now.tv_sec) % counter % (e.second.warning ? "true" : "false") % DNSAction::typeToString(e.second.action != DNSAction::Action::None ? e.second.action : g_dynBlockAction) % e.second.reason).str(); - } + setLuaNoSideEffect(); + auto slow = g_dynblockNMG.getCopy(); + struct timespec now; + gettime(&now); + boost::format fmt("%-24s %8d %8d %-10s %-20s %s\n"); + g_outputBuffer = (fmt % "What" % "Seconds" % "Blocks" % "Warning" % "Action" % "Reason").str(); + for (const auto& e : slow) { + if (now < e.second.until) { + uint64_t counter = e.second.blocks; + if (g_defaultBPFFilter && e.second.bpf) { + counter += g_defaultBPFFilter->getHits(e.first.getNetwork()); + } + g_outputBuffer += (fmt % e.first.toString() % (e.second.until.tv_sec - now.tv_sec) % counter % (e.second.warning ? "true" : "false") % DNSAction::typeToString(e.second.action != DNSAction::Action::None ? e.second.action : g_dynBlockAction) % e.second.reason).str(); + } + } + auto slow2 = g_dynblockSMT.getCopy(); + slow2.visit([&now, &fmt](const SuffixMatchTree& node) { + if (now < node.d_value.until) { + string dom("empty"); + if (!node.d_value.domain.empty()) + dom = node.d_value.domain.toString(); + g_outputBuffer += (fmt % dom % (node.d_value.until.tv_sec - now.tv_sec) % node.d_value.blocks % (node.d_value.warning ? "true" : "false") % DNSAction::typeToString(node.d_value.action != DNSAction::Action::None ? node.d_value.action : g_dynBlockAction) % node.d_value.reason).str(); } - auto slow2 = g_dynblockSMT.getCopy(); - slow2.visit([&now, &fmt](const SuffixMatchTree& node) { - if(now smt; - g_dynblockSMT.setState(smt); - }); + setLuaSideEffect(); + nmts_t nmg; + g_dynblockNMG.setState(nmg); + SuffixMatchTree smt; + g_dynblockSMT.setState(smt); + }); luaCtx.writeFunction("addDynBlocks", - [](const std::unordered_map& m, const std::string& msg, boost::optional seconds, boost::optional action) { - if (m.empty()) { - return; + [](const std::unordered_map& m, const std::string& msg, boost::optional seconds, boost::optional action) { + if (m.empty()) { + return; + } + setLuaSideEffect(); + auto slow = g_dynblockNMG.getCopy(); + struct timespec until, now; + gettime(&now); + until = now; + int actualSeconds = seconds ? *seconds : 10; + until.tv_sec += actualSeconds; + for (const auto& capair : m) { + unsigned int count = 0; + /* this legacy interface does not support ranges or ports, use DynBlockRulesGroup instead */ + AddressAndPortRange requestor(capair.first, capair.first.isIPv4() ? 32 : 128, 0); + auto got = slow.lookup(requestor); + bool expired = false; + if (got) { + if (until < got->second.until) { + // had a longer policy + continue; + } + if (now < got->second.until) { + // only inherit count on fresh query we are extending + count = got->second.blocks; + } + else { + expired = true; + } } - setLuaSideEffect(); - auto slow = g_dynblockNMG.getCopy(); - struct timespec until, now; - gettime(&now); - until=now; - int actualSeconds = seconds ? *seconds : 10; - until.tv_sec += actualSeconds; - for(const auto& capair : m) { - unsigned int count = 0; - auto got = slow.lookup(Netmask(capair.first)); - bool expired=false; - if(got) { - if(until < got->second.until) // had a longer policy - continue; - if(now < got->second.until) // only inherit count on fresh query we are extending - count=got->second.blocks; - else - expired=true; - } - DynBlock db{msg,until,DNSName(),(action ? *action : DNSAction::Action::None)}; - db.blocks=count; - if(!got || expired) - warnlog("Inserting dynamic block for %s for %d seconds: %s", capair.first.toString(), actualSeconds, msg); - slow.insert(Netmask(capair.first)).second=db; - } - g_dynblockNMG.setState(slow); - }); + DynBlock db{msg, until, DNSName(), (action ? *action : DNSAction::Action::None)}; + db.blocks = count; + if (!got || expired) { + warnlog("Inserting dynamic block for %s for %d seconds: %s", capair.first.toString(), actualSeconds, msg); + } + slow.insert(requestor).second = db; + } + g_dynblockNMG.setState(slow); + }); luaCtx.writeFunction("addDynBlockSMT", - [](const vector >&names, const std::string& msg, boost::optional seconds, boost::optional action) { - if (names.empty()) { - return; + [](const vector>& names, const std::string& msg, boost::optional seconds, boost::optional action) { + if (names.empty()) { + return; + } + setLuaSideEffect(); + auto slow = g_dynblockSMT.getCopy(); + struct timespec until, now; + gettime(&now); + until = now; + int actualSeconds = seconds ? *seconds : 10; + until.tv_sec += actualSeconds; + + for (const auto& capair : names) { + unsigned int count = 0; + DNSName domain(capair.second); + domain.makeUsLowerCase(); + auto got = slow.lookup(domain); + bool expired = false; + if (got) { + if (until < got->until) // had a longer policy + continue; + if (now < got->until) // only inherit count on fresh query we are extending + count = got->blocks; + else + expired = true; } - setLuaSideEffect(); - auto slow = g_dynblockSMT.getCopy(); - struct timespec until, now; - gettime(&now); - until=now; - int actualSeconds = seconds ? *seconds : 10; - until.tv_sec += actualSeconds; - - for(const auto& capair : names) { - unsigned int count = 0; - DNSName domain(capair.second); - domain.makeUsLowerCase(); - auto got = slow.lookup(domain); - bool expired=false; - if(got) { - if(until < got->until) // had a longer policy - continue; - if(now < got->until) // only inherit count on fresh query we are extending - count=got->blocks; - else - expired=true; - } - - DynBlock db{msg,until,domain,(action ? *action : DNSAction::Action::None)}; - db.blocks=count; - if(!got || expired) - warnlog("Inserting dynamic block for %s for %d seconds: %s", domain, actualSeconds, msg); - slow.add(domain, std::move(db)); - } - g_dynblockSMT.setState(slow); - }); + + DynBlock db{msg, until, domain, (action ? *action : DNSAction::Action::None)}; + db.blocks = count; + if (!got || expired) + warnlog("Inserting dynamic block for %s for %d seconds: %s", domain, actualSeconds, msg); + slow.add(domain, std::move(db)); + } + g_dynblockSMT.setState(slow); + }); luaCtx.writeFunction("setDynBlocksAction", [](DNSAction::Action action) { - if (!g_configurationDone) { - if (action == DNSAction::Action::Drop || action == DNSAction::Action::NoOp || action == DNSAction::Action::Nxdomain || action == DNSAction::Action::Refused || action == DNSAction::Action::Truncate || action == DNSAction::Action::NoRecurse) { - g_dynBlockAction = action; - } - else { - errlog("Dynamic blocks action can only be Drop, NoOp, NXDomain, Refused, Truncate or NoRecurse!"); - g_outputBuffer="Dynamic blocks action can only be Drop, NoOp, NXDomain, Refused, Truncate or NoRecurse!\n"; - } - } else { - g_outputBuffer="Dynamic blocks action cannot be altered at runtime!\n"; + if (!g_configurationDone) { + if (action == DNSAction::Action::Drop || action == DNSAction::Action::NoOp || action == DNSAction::Action::Nxdomain || action == DNSAction::Action::Refused || action == DNSAction::Action::Truncate || action == DNSAction::Action::NoRecurse) { + g_dynBlockAction = action; } - }); + else { + errlog("Dynamic blocks action can only be Drop, NoOp, NXDomain, Refused, Truncate or NoRecurse!"); + g_outputBuffer = "Dynamic blocks action can only be Drop, NoOp, NXDomain, Refused, Truncate or NoRecurse!\n"; + } + } + else { + g_outputBuffer = "Dynamic blocks action cannot be altered at runtime!\n"; + } + }); luaCtx.writeFunction("setDynBlocksPurgeInterval", [](unsigned int interval) { DynBlockMaintenance::s_expiredDynBlocksPurgeInterval = interval; }); luaCtx.writeFunction("addDNSCryptBind", [](const std::string& addr, const std::string& providerName, boost::variant>> certFiles, boost::variant>> keyFiles, boost::optional vars) { - if (g_configurationDone) { - g_outputBuffer="addDNSCryptBind cannot be used at runtime!\n"; - return; - } + if (g_configurationDone) { + g_outputBuffer = "addDNSCryptBind cannot be used at runtime!\n"; + return; + } #ifdef HAVE_DNSCRYPT - bool reusePort = false; - int tcpFastOpenQueueSize = 0; - int tcpListenQueueSize = 0; - size_t maxInFlightQueriesPerConn = 0; - size_t tcpMaxConcurrentConnections = 0; - std::string interface; - std::set cpus; - std::vector certKeys; + bool reusePort = false; + int tcpFastOpenQueueSize = 0; + int tcpListenQueueSize = 0; + size_t maxInFlightQueriesPerConn = 0; + size_t tcpMaxConcurrentConnections = 0; + std::string interface; + std::set cpus; + std::vector certKeys; - parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConnections); + parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConnections); - if (certFiles.type() == typeid(std::string) && keyFiles.type() == typeid(std::string)) { - auto certFile = boost::get(certFiles); - auto keyFile = boost::get(keyFiles); - certKeys.push_back({certFile, keyFile}); - } - else if (certFiles.type() == typeid(std::vector>) && keyFiles.type() == typeid(std::vector>)) { - auto certFilesVect = boost::get>>(certFiles); - auto keyFilesVect = boost::get>>(keyFiles); - if (certFilesVect.size() == keyFilesVect.size()) { - for (size_t idx = 0; idx < certFilesVect.size(); idx++) { - certKeys.push_back({certFilesVect.at(idx).second, keyFilesVect.at(idx).second}); - } - } - else { - errlog("Error, mismatching number of certificates and keys in call to addDNSCryptBind!"); - g_outputBuffer="Error, mismatching number of certificates and keys in call to addDNSCryptBind()!"; - return; + if (certFiles.type() == typeid(std::string) && keyFiles.type() == typeid(std::string)) { + auto certFile = boost::get(certFiles); + auto keyFile = boost::get(keyFiles); + certKeys.push_back({certFile, keyFile}); + } + else if (certFiles.type() == typeid(std::vector>) && keyFiles.type() == typeid(std::vector>)) { + auto certFilesVect = boost::get>>(certFiles); + auto keyFilesVect = boost::get>>(keyFiles); + if (certFilesVect.size() == keyFilesVect.size()) { + for (size_t idx = 0; idx < certFilesVect.size(); idx++) { + certKeys.push_back({certFilesVect.at(idx).second, keyFilesVect.at(idx).second}); } } else { - errlog("Error, mismatching number of certificates and keys in call to addDNSCryptBind()!"); - g_outputBuffer="Error, mismatching number of certificates and keys in call to addDNSCryptBind()!"; + errlog("Error, mismatching number of certificates and keys in call to addDNSCryptBind!"); + g_outputBuffer = "Error, mismatching number of certificates and keys in call to addDNSCryptBind()!"; return; } + } + else { + errlog("Error, mismatching number of certificates and keys in call to addDNSCryptBind()!"); + g_outputBuffer = "Error, mismatching number of certificates and keys in call to addDNSCryptBind()!"; + return; + } - try { - auto ctx = std::make_shared(providerName, certKeys); - - /* UDP */ - auto cs = std::unique_ptr(new ClientState(ComboAddress(addr, 443), false, reusePort, tcpFastOpenQueueSize, interface, cpus)); - cs->dnscryptCtx = ctx; - g_dnsCryptLocals.push_back(ctx); - g_frontends.push_back(std::move(cs)); - - /* TCP */ - cs = std::unique_ptr(new ClientState(ComboAddress(addr, 443), true, reusePort, tcpFastOpenQueueSize, interface, cpus)); - cs->dnscryptCtx = ctx; - if (tcpListenQueueSize > 0) { - cs->tcpListenQueueSize = tcpListenQueueSize; - } - if (maxInFlightQueriesPerConn > 0) { - cs->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; - } - if (tcpMaxConcurrentConnections > 0) { - cs->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; - } + try { + auto ctx = std::make_shared(providerName, certKeys); + + /* UDP */ + auto cs = std::make_unique(ComboAddress(addr, 443), false, reusePort, tcpFastOpenQueueSize, interface, cpus); + cs->dnscryptCtx = ctx; + g_dnsCryptLocals.push_back(ctx); + g_frontends.push_back(std::move(cs)); - g_frontends.push_back(std::move(cs)); + /* TCP */ + cs = std::make_unique(ComboAddress(addr, 443), true, reusePort, tcpFastOpenQueueSize, interface, cpus); + cs->dnscryptCtx = ctx; + if (tcpListenQueueSize > 0) { + cs->tcpListenQueueSize = tcpListenQueueSize; + } + if (maxInFlightQueriesPerConn > 0) { + cs->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; } - catch(std::exception& e) { - errlog(e.what()); - g_outputBuffer="Error: "+string(e.what())+"\n"; + if (tcpMaxConcurrentConnections > 0) { + cs->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; } + + g_frontends.push_back(std::move(cs)); + } + catch (std::exception& e) { + errlog(e.what()); + g_outputBuffer = "Error: " + string(e.what()) + "\n"; + } #else - g_outputBuffer="Error: DNSCrypt support is not enabled.\n"; + g_outputBuffer = "Error: DNSCrypt support is not enabled.\n"; #endif - - }); + }); luaCtx.writeFunction("showDNSCryptBinds", []() { - setLuaNoSideEffect(); + setLuaNoSideEffect(); #ifdef HAVE_DNSCRYPT - ostringstream ret; - boost::format fmt("%1$-3d %2% %|25t|%3$-20.20s"); - ret << (fmt % "#" % "Address" % "Provider Name") << endl; - size_t idx = 0; - - std::unordered_set> contexts; - for (const auto& frontend : g_frontends) { - const std::shared_ptr ctx = frontend->dnscryptCtx; - if (!ctx || contexts.count(ctx) != 0) { - continue; - } - contexts.insert(ctx); - ret<< (fmt % idx % frontend->local.toStringWithPort() % ctx->getProviderName()) << endl; - idx++; - } + ostringstream ret; + boost::format fmt("%1$-3d %2% %|25t|%3$-20.20s"); + ret << (fmt % "#" % "Address" % "Provider Name") << endl; + size_t idx = 0; + + std::unordered_set> contexts; + for (const auto& frontend : g_frontends) { + const std::shared_ptr ctx = frontend->dnscryptCtx; + if (!ctx || contexts.count(ctx) != 0) { + continue; + } + contexts.insert(ctx); + ret << (fmt % idx % frontend->local.toStringWithPort() % ctx->getProviderName()) << endl; + idx++; + } - g_outputBuffer=ret.str(); + g_outputBuffer = ret.str(); #else - g_outputBuffer="Error: DNSCrypt support is not enabled.\n"; + g_outputBuffer = "Error: DNSCrypt support is not enabled.\n"; #endif - }); + }); luaCtx.writeFunction("getDNSCryptBind", [](size_t idx) { - setLuaNoSideEffect(); + setLuaNoSideEffect(); #ifdef HAVE_DNSCRYPT - std::shared_ptr ret = nullptr; - if (idx < g_dnsCryptLocals.size()) { - ret = g_dnsCryptLocals.at(idx); - } - return ret; + std::shared_ptr ret = nullptr; + if (idx < g_dnsCryptLocals.size()) { + ret = g_dnsCryptLocals.at(idx); + } + return ret; #else - g_outputBuffer="Error: DNSCrypt support is not enabled.\n"; + g_outputBuffer = "Error: DNSCrypt support is not enabled.\n"; #endif - }); + }); luaCtx.writeFunction("getDNSCryptBindCount", []() { - setLuaNoSideEffect(); - return g_dnsCryptLocals.size(); - }); + setLuaNoSideEffect(); + return g_dnsCryptLocals.size(); + }); luaCtx.writeFunction("generateDNSCryptProviderKeys", [client](const std::string& publicKeyFile, const std::string privateKeyFile) { - setLuaNoSideEffect(); + setLuaNoSideEffect(); #ifdef HAVE_DNSCRYPT - if (client) { - return; - } - unsigned char publicKey[DNSCRYPT_PROVIDER_PUBLIC_KEY_SIZE]; - unsigned char privateKey[DNSCRYPT_PROVIDER_PRIVATE_KEY_SIZE]; - sodium_mlock(privateKey, sizeof(privateKey)); + if (client) { + return; + } + unsigned char publicKey[DNSCRYPT_PROVIDER_PUBLIC_KEY_SIZE]; + unsigned char privateKey[DNSCRYPT_PROVIDER_PRIVATE_KEY_SIZE]; + sodium_mlock(privateKey, sizeof(privateKey)); - try { - DNSCryptContext::generateProviderKeys(publicKey, privateKey); + try { + DNSCryptContext::generateProviderKeys(publicKey, privateKey); - ofstream pubKStream(publicKeyFile); - pubKStream.write((char*) publicKey, sizeof(publicKey)); - pubKStream.close(); + ofstream pubKStream(publicKeyFile); + pubKStream.write((char*)publicKey, sizeof(publicKey)); + pubKStream.close(); - ofstream privKStream(privateKeyFile); - privKStream.write((char*) privateKey, sizeof(privateKey)); - privKStream.close(); + ofstream privKStream(privateKeyFile); + privKStream.write((char*)privateKey, sizeof(privateKey)); + privKStream.close(); - g_outputBuffer="Provider fingerprint is: " + DNSCryptContext::getProviderFingerprint(publicKey) + "\n"; - } - catch(std::exception& e) { - errlog(e.what()); - g_outputBuffer="Error: "+string(e.what())+"\n"; - } + g_outputBuffer = "Provider fingerprint is: " + DNSCryptContext::getProviderFingerprint(publicKey) + "\n"; + } + catch (std::exception& e) { + errlog(e.what()); + g_outputBuffer = "Error: " + string(e.what()) + "\n"; + } - sodium_memzero(privateKey, sizeof(privateKey)); - sodium_munlock(privateKey, sizeof(privateKey)); + sodium_memzero(privateKey, sizeof(privateKey)); + sodium_munlock(privateKey, sizeof(privateKey)); #else - g_outputBuffer="Error: DNSCrypt support is not enabled.\n"; + g_outputBuffer = "Error: DNSCrypt support is not enabled.\n"; #endif - }); + }); luaCtx.writeFunction("printDNSCryptProviderFingerprint", [](const std::string& publicKeyFile) { - setLuaNoSideEffect(); + setLuaNoSideEffect(); #ifdef HAVE_DNSCRYPT - unsigned char publicKey[DNSCRYPT_PROVIDER_PUBLIC_KEY_SIZE]; + unsigned char publicKey[DNSCRYPT_PROVIDER_PUBLIC_KEY_SIZE]; - try { - ifstream file(publicKeyFile); - file.read((char *) &publicKey, sizeof(publicKey)); + try { + ifstream file(publicKeyFile); + file.read((char*)&publicKey, sizeof(publicKey)); - if (file.fail()) - throw std::runtime_error("Invalid dnscrypt provider public key file " + publicKeyFile); + if (file.fail()) + throw std::runtime_error("Invalid dnscrypt provider public key file " + publicKeyFile); - file.close(); - g_outputBuffer="Provider fingerprint is: " + DNSCryptContext::getProviderFingerprint(publicKey) + "\n"; - } - catch(std::exception& e) { - errlog(e.what()); - g_outputBuffer="Error: "+string(e.what())+"\n"; - } + file.close(); + g_outputBuffer = "Provider fingerprint is: " + DNSCryptContext::getProviderFingerprint(publicKey) + "\n"; + } + catch (std::exception& e) { + errlog(e.what()); + g_outputBuffer = "Error: " + string(e.what()) + "\n"; + } #else - g_outputBuffer="Error: DNSCrypt support is not enabled.\n"; + g_outputBuffer = "Error: DNSCrypt support is not enabled.\n"; #endif - }); + }); #ifdef HAVE_DNSCRYPT luaCtx.writeFunction("generateDNSCryptCertificate", [client](const std::string& providerPrivateKeyFile, const std::string& certificateFile, const std::string privateKeyFile, uint32_t serial, time_t begin, time_t end, boost::optional version) { - setLuaNoSideEffect(); - if (client) { - return; - } - DNSCryptPrivateKey privateKey; - DNSCryptCert cert; + setLuaNoSideEffect(); + if (client) { + return; + } + DNSCryptPrivateKey privateKey; + DNSCryptCert cert; - try { - if (generateDNSCryptCertificate(providerPrivateKeyFile, serial, begin, end, version ? *version : DNSCryptExchangeVersion::VERSION1, cert, privateKey)) { - privateKey.saveToFile(privateKeyFile); - DNSCryptContext::saveCertFromFile(cert, certificateFile); - } + try { + if (generateDNSCryptCertificate(providerPrivateKeyFile, serial, begin, end, version ? *version : DNSCryptExchangeVersion::VERSION1, cert, privateKey)) { + privateKey.saveToFile(privateKeyFile); + DNSCryptContext::saveCertFromFile(cert, certificateFile); } - catch(const std::exception& e) { - errlog(e.what()); - g_outputBuffer="Error: "+string(e.what())+"\n"; - } - }); + } + catch (const std::exception& e) { + errlog(e.what()); + g_outputBuffer = "Error: " + string(e.what()) + "\n"; + } + }); #endif luaCtx.writeFunction("showPools", []() { - setLuaNoSideEffect(); - try { - ostringstream ret; - boost::format fmt("%1$-20.20s %|25t|%2$20s %|25t|%3$20s %|50t|%4%" ); - // 1 2 3 4 - ret << (fmt % "Name" % "Cache" % "ServerPolicy" % "Servers" ) << endl; - - const auto localPools = g_pools.getCopy(); - for (const auto& entry : localPools) { - const string& name = entry.first; - const std::shared_ptr pool = entry.second; - string cache = pool->packetCache != nullptr ? pool->packetCache->toString() : ""; - string policy = g_policy.getLocal()->getName(); - if (pool->policy != nullptr) { - policy = pool->policy->getName(); + setLuaNoSideEffect(); + try { + ostringstream ret; + boost::format fmt("%1$-20.20s %|25t|%2$20s %|25t|%3$20s %|50t|%4%"); + // 1 2 3 4 + ret << (fmt % "Name" % "Cache" % "ServerPolicy" % "Servers") << endl; + + const auto localPools = g_pools.getCopy(); + for (const auto& entry : localPools) { + const string& name = entry.first; + const std::shared_ptr pool = entry.second; + string cache = pool->packetCache != nullptr ? pool->packetCache->toString() : ""; + string policy = g_policy.getLocal()->getName(); + if (pool->policy != nullptr) { + policy = pool->policy->getName(); + } + string servers; + + const auto poolServers = pool->getServers(); + for (const auto& server : *poolServers) { + if (!servers.empty()) { + servers += ", "; } - string servers; - - const auto poolServers = pool->getServers(); - for (const auto& server: *poolServers) { - if (!servers.empty()) { - servers += ", "; - } - if (!server.second->getName().empty()) { - servers += server.second->getName(); - servers += " "; - } - servers += server.second->remote.toStringWithPort(); + if (!server.second->getName().empty()) { + servers += server.second->getName(); + servers += " "; } - - ret << (fmt % name % cache % policy % servers) << endl; + servers += server.second->remote.toStringWithPort(); } - g_outputBuffer=ret.str(); - }catch(std::exception& e) { g_outputBuffer=e.what(); throw; } - }); - luaCtx.writeFunction("getPool", [client](const string& poolName) { - if (client) { - return std::make_shared(); + ret << (fmt % name % cache % policy % servers) << endl; } - auto localPools = g_pools.getCopy(); - std::shared_ptr pool = createPoolIfNotExists(localPools, poolName); - g_pools.setState(localPools); - return pool; - }); + g_outputBuffer = ret.str(); + } + catch (std::exception& e) { + g_outputBuffer = e.what(); + throw; + } + }); + + luaCtx.writeFunction("getPool", [client](const string& poolName) { + if (client) { + return std::make_shared(); + } + auto localPools = g_pools.getCopy(); + std::shared_ptr pool = createPoolIfNotExists(localPools, poolName); + g_pools.setState(localPools); + return pool; + }); - luaCtx.writeFunction("setVerboseHealthChecks", [](bool verbose) { g_verboseHealthChecks=verbose; }); - luaCtx.writeFunction("setStaleCacheEntriesTTL", [](uint32_t ttl) { g_staleCacheEntriesTTL = ttl; }); + luaCtx.writeFunction("setVerboseHealthChecks", [](bool verbose) { g_verboseHealthChecks = verbose; }); + luaCtx.writeFunction("setStaleCacheEntriesTTL", [](uint64_t ttl) { + checkParameterBound("setStaleCacheEntriesTTL", ttl, std::numeric_limits::max()); + g_staleCacheEntriesTTL = ttl; + }); luaCtx.writeFunction("showBinds", []() { - setLuaNoSideEffect(); - try { - ostringstream ret; - boost::format fmt("%1$-3d %2$-20.20s %|35t|%3$-20.20s %|57t|%4%" ); - // 1 2 3 4 - ret << (fmt % "#" % "Address" % "Protocol" % "Queries" ) << endl; - - size_t counter = 0; - for (const auto& front : g_frontends) { - ret << (fmt % counter % front->local.toStringWithPort() % front->getType() % front->queries) << endl; - counter++; - } - g_outputBuffer=ret.str(); - }catch(std::exception& e) { g_outputBuffer=e.what(); throw; } - }); + setLuaNoSideEffect(); + try { + ostringstream ret; + boost::format fmt("%1$-3d %2$-20.20s %|35t|%3$-20.20s %|57t|%4%"); + // 1 2 3 4 + ret << (fmt % "#" % "Address" % "Protocol" % "Queries") << endl; - luaCtx.writeFunction("getBind", [](size_t num) { - setLuaNoSideEffect(); - ClientState* ret = nullptr; - if(num < g_frontends.size()) { - ret=g_frontends[num].get(); + size_t counter = 0; + for (const auto& front : g_frontends) { + ret << (fmt % counter % front->local.toStringWithPort() % front->getType() % front->queries) << endl; + counter++; } - return ret; - }); + g_outputBuffer = ret.str(); + } + catch (std::exception& e) { + g_outputBuffer = e.what(); + throw; + } + }); + + luaCtx.writeFunction("getBind", [](size_t num) { + setLuaNoSideEffect(); + ClientState* ret = nullptr; + if (num < g_frontends.size()) { + ret = g_frontends[num].get(); + } + return ret; + }); luaCtx.writeFunction("getBindCount", []() { - setLuaNoSideEffect(); - return g_frontends.size(); - }); + setLuaNoSideEffect(); + return g_frontends.size(); + }); luaCtx.writeFunction("help", [](boost::optional command) { - setLuaNoSideEffect(); - g_outputBuffer = ""; - for (const auto& keyword : g_consoleKeywords) { - if (!command) { - g_outputBuffer += keyword.toString() + "\n"; - } - else if (keyword.name == command) { - g_outputBuffer = keyword.toString() + "\n"; - return; - } - } - if (command) { - g_outputBuffer = "Nothing found for " + *command + "\n"; + setLuaNoSideEffect(); + g_outputBuffer = ""; + for (const auto& keyword : g_consoleKeywords) { + if (!command) { + g_outputBuffer += keyword.toString() + "\n"; + } + else if (keyword.name == command) { + g_outputBuffer = keyword.toString() + "\n"; + return; } - }); + } + if (command) { + g_outputBuffer = "Nothing found for " + *command + "\n"; + } + }); luaCtx.writeFunction("showVersion", []() { - setLuaNoSideEffect(); - g_outputBuffer = "dnsdist " + std::string(VERSION) + "\n"; - }); + setLuaNoSideEffect(); + g_outputBuffer = "dnsdist " + std::string(VERSION) + "\n"; + }); luaCtx.writeFunction("showSecurityStatus", []() { - setLuaNoSideEffect(); - g_outputBuffer = std::to_string(g_stats.securityStatus) + "\n"; - }); + setLuaNoSideEffect(); + g_outputBuffer = std::to_string(g_stats.securityStatus) + "\n"; + }); #ifdef HAVE_EBPF luaCtx.writeFunction("setDefaultBPFFilter", [](std::shared_ptr bpf) { - if (g_configurationDone) { - g_outputBuffer="setDefaultBPFFilter() cannot be used at runtime!\n"; - return; - } - g_defaultBPFFilter = bpf; - }); + if (g_configurationDone) { + g_outputBuffer = "setDefaultBPFFilter() cannot be used at runtime!\n"; + return; + } + g_defaultBPFFilter = bpf; + }); luaCtx.writeFunction("registerDynBPFFilter", [](std::shared_ptr dbpf) { - if (dbpf) { - g_dynBPFFilters.push_back(dbpf); - } - }); + if (dbpf) { + g_dynBPFFilters.push_back(dbpf); + } + }); luaCtx.writeFunction("unregisterDynBPFFilter", [](std::shared_ptr dbpf) { - if (dbpf) { - for (auto it = g_dynBPFFilters.begin(); it != g_dynBPFFilters.end(); it++) { - if (*it == dbpf) { - g_dynBPFFilters.erase(it); - break; - } + if (dbpf) { + for (auto it = g_dynBPFFilters.begin(); it != g_dynBPFFilters.end(); it++) { + if (*it == dbpf) { + g_dynBPFFilters.erase(it); + break; } } - }); + } + }); - luaCtx.writeFunction("addBPFFilterDynBlocks", [](const std::unordered_map& m, std::shared_ptr dynbpf, boost::optional seconds, boost::optional msg) { - if (!dynbpf) { - return; - } - setLuaSideEffect(); - struct timespec until, now; - clock_gettime(CLOCK_MONOTONIC, &now); - until=now; - int actualSeconds = seconds ? *seconds : 10; - until.tv_sec += actualSeconds; - for(const auto& capair : m) { - if (dynbpf->block(capair.first, until)) { - warnlog("Inserting eBPF dynamic block for %s for %d seconds: %s", capair.first.toString(), actualSeconds, msg ? *msg : ""); - } + luaCtx.writeFunction("addBPFFilterDynBlocks", [](const std::unordered_map& m, std::shared_ptr dynbpf, boost::optional seconds, boost::optional msg) { + if (!dynbpf) { + return; + } + setLuaSideEffect(); + struct timespec until, now; + clock_gettime(CLOCK_MONOTONIC, &now); + until = now; + int actualSeconds = seconds ? *seconds : 10; + until.tv_sec += actualSeconds; + for (const auto& capair : m) { + if (dynbpf->block(capair.first, until)) { + warnlog("Inserting eBPF dynamic block for %s for %d seconds: %s", capair.first.toString(), actualSeconds, msg ? *msg : ""); } - }); + } + }); #endif /* HAVE_EBPF */ - luaCtx.writeFunction()>("getStatisticsCounters", []() { - setLuaNoSideEffect(); - std::unordered_map res; - for(const auto& entry : g_stats.entries) { - if(const auto& val = boost::get(&entry.second)) - res[entry.first] = (*val)->load(); - } - return res; - }); + luaCtx.writeFunction()>("getStatisticsCounters", []() { + setLuaNoSideEffect(); + std::unordered_map res; + for (const auto& entry : g_stats.entries) { + if (const auto& val = boost::get(&entry.second)) + res[entry.first] = (*val)->load(); + } + return res; + }); luaCtx.writeFunction("includeDirectory", [&luaCtx](const std::string& dirname) { - if (g_configurationDone) { - errlog("includeDirectory() cannot be used at runtime!"); - g_outputBuffer="includeDirectory() cannot be used at runtime!\n"; - return; - } + if (g_configurationDone) { + errlog("includeDirectory() cannot be used at runtime!"); + g_outputBuffer = "includeDirectory() cannot be used at runtime!\n"; + return; + } - if (g_included) { - errlog("includeDirectory() cannot be used recursively!"); - g_outputBuffer="includeDirectory() cannot be used recursively!\n"; - return; - } + if (g_included) { + errlog("includeDirectory() cannot be used recursively!"); + g_outputBuffer = "includeDirectory() cannot be used recursively!\n"; + return; + } - g_included = true; - struct stat st; - if (stat(dirname.c_str(), &st)) { - errlog("The included directory %s does not exist!", dirname.c_str()); - g_outputBuffer="The included directory " + dirname + " does not exist!"; - return; - } + struct stat st; + if (stat(dirname.c_str(), &st)) { + errlog("The included directory %s does not exist!", dirname.c_str()); + g_outputBuffer = "The included directory " + dirname + " does not exist!"; + return; + } - if (!S_ISDIR(st.st_mode)) { - errlog("The included directory %s is not a directory!", dirname.c_str()); - g_outputBuffer="The included directory " + dirname + " is not a directory!"; - return; - } + if (!S_ISDIR(st.st_mode)) { + errlog("The included directory %s is not a directory!", dirname.c_str()); + g_outputBuffer = "The included directory " + dirname + " is not a directory!"; + return; + } - DIR *dirp; - struct dirent *ent; - std::list files; - if (!(dirp = opendir(dirname.c_str()))) { - errlog("Error opening the included directory %s!", dirname.c_str()); - g_outputBuffer="Error opening the included directory " + dirname + "!"; - return; + DIR* dirp; + struct dirent* ent; + std::list files; + if (!(dirp = opendir(dirname.c_str()))) { + errlog("Error opening the included directory %s!", dirname.c_str()); + g_outputBuffer = "Error opening the included directory " + dirname + "!"; + return; + } + + while ((ent = readdir(dirp)) != NULL) { + if (ent->d_name[0] == '.') { + continue; } - while((ent = readdir(dirp)) != NULL) { - if (ent->d_name[0] == '.') { + if (boost::ends_with(ent->d_name, ".conf")) { + std::ostringstream namebuf; + namebuf << dirname.c_str() << "/" << ent->d_name; + + if (stat(namebuf.str().c_str(), &st) || !S_ISREG(st.st_mode)) { continue; } - if (boost::ends_with(ent->d_name, ".conf")) { - std::ostringstream namebuf; - namebuf << dirname.c_str() << "/" << ent->d_name; - - if (stat(namebuf.str().c_str(), &st) || !S_ISREG(st.st_mode)) { - continue; - } - - files.push_back(namebuf.str()); - } + files.push_back(namebuf.str()); } + } - closedir(dirp); - files.sort(); + closedir(dirp); + files.sort(); - for (auto file = files.begin(); file != files.end(); ++file) { - std::ifstream ifs(*file); - if (!ifs) { - warnlog("Unable to read configuration from '%s'", *file); - } else { - vinfolog("Read configuration from '%s'", *file); - } + g_included = true; + for (auto file = files.begin(); file != files.end(); ++file) { + std::ifstream ifs(*file); + if (!ifs) { + warnlog("Unable to read configuration from '%s'", *file); + } + else { + vinfolog("Read configuration from '%s'", *file); + } + + try { luaCtx.executeCode(ifs); } + catch (...) { + g_included = false; + throw; + } - g_included = false; - }); + luaCtx.executeCode(ifs); + } + + g_included = false; + }); luaCtx.writeFunction("setAPIWritable", [](bool writable, boost::optional apiConfigDir) { - setLuaSideEffect(); - g_apiReadWrite = writable; - if (apiConfigDir) { - if (!(*apiConfigDir).empty()) { - g_apiConfigDirectory = *apiConfigDir; - } - else { - errlog("The API configuration directory value cannot be empty!"); - g_outputBuffer="The API configuration directory value cannot be empty!"; - } + setLuaSideEffect(); + g_apiReadWrite = writable; + if (apiConfigDir) { + if (!(*apiConfigDir).empty()) { + g_apiConfigDirectory = *apiConfigDir; } - }); + else { + errlog("The API configuration directory value cannot be empty!"); + g_outputBuffer = "The API configuration directory value cannot be empty!"; + } + } + }); luaCtx.writeFunction("setServFailWhenNoServer", [](bool servfail) { - setLuaSideEffect(); - g_servFailOnNoPolicy = servfail; - }); + setLuaSideEffect(); + g_servFailOnNoPolicy = servfail; + }); luaCtx.writeFunction("setRoundRobinFailOnNoServer", [](bool fail) { - setLuaSideEffect(); - g_roundrobinFailOnNoServer = fail; - }); + setLuaSideEffect(); + g_roundrobinFailOnNoServer = fail; + }); luaCtx.writeFunction("setConsistentHashingBalancingFactor", [](double factor) { - setLuaSideEffect(); - if (factor >= 1.0) { - g_consistentHashBalancingFactor = factor; - } - else { - errlog("Invalid value passed to setConsistentHashingBalancingFactor()!"); - g_outputBuffer="Invalid value passed to setConsistentHashingBalancingFactor()!\n"; - return; - } - }); + setLuaSideEffect(); + if (factor >= 1.0) { + g_consistentHashBalancingFactor = factor; + } + else { + errlog("Invalid value passed to setConsistentHashingBalancingFactor()!"); + g_outputBuffer = "Invalid value passed to setConsistentHashingBalancingFactor()!\n"; + return; + } + }); luaCtx.writeFunction("setWeightedBalancingFactor", [](double factor) { - setLuaSideEffect(); - if (factor >= 1.0) { - g_weightedBalancingFactor = factor; - } - else { - errlog("Invalid value passed to setWeightedBalancingFactor()!"); - g_outputBuffer="Invalid value passed to setWeightedBalancingFactor()!\n"; - return; - } - }); + setLuaSideEffect(); + if (factor >= 1.0) { + g_weightedBalancingFactor = factor; + } + else { + errlog("Invalid value passed to setWeightedBalancingFactor()!"); + g_outputBuffer = "Invalid value passed to setWeightedBalancingFactor()!\n"; + return; + } + }); luaCtx.writeFunction("setRingBuffersSize", [client](size_t capacity, boost::optional numberOfShards) { - setLuaSideEffect(); - if (g_configurationDone) { - errlog("setRingBuffersSize() cannot be used at runtime!"); - g_outputBuffer="setRingBuffersSize() cannot be used at runtime!\n"; - return; - } - if (!client) { - g_rings.setCapacity(capacity, numberOfShards ? *numberOfShards : 10); - } - else { - g_rings.setCapacity(0, 1); - } - }); + setLuaSideEffect(); + if (g_configurationDone) { + errlog("setRingBuffersSize() cannot be used at runtime!"); + g_outputBuffer = "setRingBuffersSize() cannot be used at runtime!\n"; + return; + } + if (!client) { + g_rings.setCapacity(capacity, numberOfShards ? *numberOfShards : 10); + } + else { + g_rings.setCapacity(0, 1); + } + }); luaCtx.writeFunction("setRingBuffersLockRetries", [](size_t retries) { - setLuaSideEffect(); - g_rings.setNumberOfLockRetries(retries); - }); + setLuaSideEffect(); + g_rings.setNumberOfLockRetries(retries); + }); - luaCtx.writeFunction("setWHashedPertubation", [](uint32_t pertub) { - setLuaSideEffect(); - g_hashperturb = pertub; - }); + luaCtx.writeFunction("setWHashedPertubation", [](uint64_t perturb) { + setLuaSideEffect(); + checkParameterBound("setWHashedPertubation", perturb, std::numeric_limits::max()); + g_hashperturb = perturb; + }); luaCtx.writeFunction("setTCPInternalPipeBufferSize", [](size_t size) { g_tcpInternalPipeBufferSize = size; }); - luaCtx.writeFunction("snmpAgent", [client,configCheck](bool enableTraps, boost::optional daemonSocket) { - if(client || configCheck) - return; + luaCtx.writeFunction("snmpAgent", [client, configCheck](bool enableTraps, boost::optional daemonSocket) { + if (client || configCheck) + return; #ifdef HAVE_NET_SNMP - if (g_configurationDone) { - errlog("snmpAgent() cannot be used at runtime!"); - g_outputBuffer="snmpAgent() cannot be used at runtime!\n"; - return; - } + if (g_configurationDone) { + errlog("snmpAgent() cannot be used at runtime!"); + g_outputBuffer = "snmpAgent() cannot be used at runtime!\n"; + return; + } - if (g_snmpEnabled) { - errlog("snmpAgent() cannot be used twice!"); - g_outputBuffer="snmpAgent() cannot be used twice!\n"; - return; - } + if (g_snmpEnabled) { + errlog("snmpAgent() cannot be used twice!"); + g_outputBuffer = "snmpAgent() cannot be used twice!\n"; + return; + } - g_snmpEnabled = true; - g_snmpTrapsEnabled = enableTraps; - g_snmpAgent = new DNSDistSNMPAgent("dnsdist", daemonSocket ? *daemonSocket : std::string()); + g_snmpEnabled = true; + g_snmpTrapsEnabled = enableTraps; + g_snmpAgent = new DNSDistSNMPAgent("dnsdist", daemonSocket ? *daemonSocket : std::string()); #else errlog("NET SNMP support is required to use snmpAgent()"); - g_outputBuffer="NET SNMP support is required to use snmpAgent()\n"; + g_outputBuffer = "NET SNMP support is required to use snmpAgent()\n"; #endif /* HAVE_NET_SNMP */ - }); + }); luaCtx.writeFunction("sendCustomTrap", [](const std::string& str) { - if (g_snmpAgent && g_snmpTrapsEnabled) { - g_snmpAgent->sendCustomTrap(str); - } - }); + if (g_snmpAgent && g_snmpTrapsEnabled) { + g_snmpAgent->sendCustomTrap(str); + } + }); - luaCtx.writeFunction("setServerPolicy", [](ServerPolicy policy) { - setLuaSideEffect(); - g_policy.setState(policy); - }); + luaCtx.writeFunction("setServerPolicy", [](const ServerPolicy& policy) { + setLuaSideEffect(); + g_policy.setState(policy); + }); luaCtx.writeFunction("setServerPolicyLua", [](string name, ServerPolicy::policyfunc_t policy) { - setLuaSideEffect(); - g_policy.setState(ServerPolicy{name, policy, true}); - }); + setLuaSideEffect(); + g_policy.setState(ServerPolicy{name, policy, true}); + }); luaCtx.writeFunction("setServerPolicyLuaFFI", [](string name, ServerPolicy::ffipolicyfunc_t policy) { - setLuaSideEffect(); - auto pol = ServerPolicy(name, policy); - g_policy.setState(std::move(pol)); - }); + setLuaSideEffect(); + auto pol = ServerPolicy(name, policy); + g_policy.setState(std::move(pol)); + }); luaCtx.writeFunction("setServerPolicyLuaFFIPerThread", [](string name, const std::string& policyCode) { - setLuaSideEffect(); - auto pol = ServerPolicy(name, policyCode); - g_policy.setState(std::move(pol)); - }); + setLuaSideEffect(); + auto pol = ServerPolicy(name, policyCode); + g_policy.setState(std::move(pol)); + }); luaCtx.writeFunction("showServerPolicy", []() { - setLuaSideEffect(); - g_outputBuffer=g_policy.getLocal()->getName()+"\n"; - }); + setLuaSideEffect(); + g_outputBuffer = g_policy.getLocal()->getName() + "\n"; + }); luaCtx.writeFunction("setPoolServerPolicy", [](ServerPolicy policy, string pool) { - setLuaSideEffect(); - auto localPools = g_pools.getCopy(); - setPoolPolicy(localPools, pool, std::make_shared(policy)); - g_pools.setState(localPools); - }); + setLuaSideEffect(); + auto localPools = g_pools.getCopy(); + setPoolPolicy(localPools, pool, std::make_shared(policy)); + g_pools.setState(localPools); + }); luaCtx.writeFunction("setPoolServerPolicyLua", [](string name, ServerPolicy::policyfunc_t policy, string pool) { - setLuaSideEffect(); - auto localPools = g_pools.getCopy(); - setPoolPolicy(localPools, pool, std::make_shared(ServerPolicy{name, policy, true})); - g_pools.setState(localPools); - }); + setLuaSideEffect(); + auto localPools = g_pools.getCopy(); + setPoolPolicy(localPools, pool, std::make_shared(ServerPolicy{name, policy, true})); + g_pools.setState(localPools); + }); luaCtx.writeFunction("setPoolServerPolicyLuaFFI", [](string name, ServerPolicy::ffipolicyfunc_t policy, string pool) { - setLuaSideEffect(); - auto localPools = g_pools.getCopy(); - setPoolPolicy(localPools, pool, std::make_shared(ServerPolicy{name, policy})); - g_pools.setState(localPools); - }); + setLuaSideEffect(); + auto localPools = g_pools.getCopy(); + setPoolPolicy(localPools, pool, std::make_shared(ServerPolicy{name, policy})); + g_pools.setState(localPools); + }); luaCtx.writeFunction("setPoolServerPolicyLuaFFIPerThread", [](string name, const std::string& policyCode, string pool) { - setLuaSideEffect(); - auto localPools = g_pools.getCopy(); - setPoolPolicy(localPools, pool, std::make_shared(ServerPolicy{name, policyCode})); - g_pools.setState(localPools); - }); + setLuaSideEffect(); + auto localPools = g_pools.getCopy(); + setPoolPolicy(localPools, pool, std::make_shared(ServerPolicy{name, policyCode})); + g_pools.setState(localPools); + }); luaCtx.writeFunction("showPoolServerPolicy", [](string pool) { - setLuaSideEffect(); - auto localPools = g_pools.getCopy(); - auto poolObj = getPool(localPools, pool); - if (poolObj->policy == nullptr) { - g_outputBuffer=g_policy.getLocal()->getName()+"\n"; - } else { - g_outputBuffer=poolObj->policy->getName()+"\n"; - } - }); + setLuaSideEffect(); + auto localPools = g_pools.getCopy(); + auto poolObj = getPool(localPools, pool); + if (poolObj->policy == nullptr) { + g_outputBuffer = g_policy.getLocal()->getName() + "\n"; + } + else { + g_outputBuffer = poolObj->policy->getName() + "\n"; + } + }); - luaCtx.writeFunction("setTCPDownstreamCleanupInterval", [](uint16_t interval) { - setLuaSideEffect(); - DownstreamConnectionsManager::setCleanupInterval(interval); - }); + luaCtx.writeFunction("setTCPDownstreamCleanupInterval", [](uint64_t interval) { + setLuaSideEffect(); + checkParameterBound("setTCPDownstreamCleanupInterval", interval); + DownstreamTCPConnectionsManager::setCleanupInterval(interval); + }); + + luaCtx.writeFunction("setDoHDownstreamCleanupInterval", [](uint64_t interval) { + setLuaSideEffect(); + checkParameterBound("setDoHDownstreamCleanupInterval", interval); + setDoHDownstreamCleanupInterval(interval); + }); + + luaCtx.writeFunction("setTCPDownstreamMaxIdleTime", [](uint64_t max) { + setLuaSideEffect(); + checkParameterBound("setTCPDownstreamMaxIdleTime", max); + DownstreamTCPConnectionsManager::setMaxIdleTime(max); + }); + + luaCtx.writeFunction("setDoHDownstreamMaxIdleTime", [](uint64_t max) { + setLuaSideEffect(); + checkParameterBound("setDoHDownstreamMaxIdleTime", max); + setDoHDownstreamMaxIdleTime(max); + }); luaCtx.writeFunction("setConsoleConnectionsLogging", [](bool enabled) { - g_logConsoleConnections = enabled; - }); + g_logConsoleConnections = enabled; + }); - luaCtx.writeFunction("setConsoleOutputMaxMsgSize", [](uint32_t size) { - g_consoleOutputMsgMaxSize = size; - }); + luaCtx.writeFunction("setConsoleOutputMaxMsgSize", [](uint64_t size) { + checkParameterBound("setConsoleOutputMaxMsgSize", size, std::numeric_limits::max()); + g_consoleOutputMsgMaxSize = size; + }); - luaCtx.writeFunction("setProxyProtocolACL", [](boost::variant>> inp) { + luaCtx.writeFunction("setProxyProtocolACL", [](boost::variant>> inp) { if (g_configurationDone) { errlog("setProxyProtocolACL() cannot be used at runtime!"); - g_outputBuffer="setProxyProtocolACL() cannot be used at runtime!\n"; + g_outputBuffer = "setProxyProtocolACL() cannot be used at runtime!\n"; return; } setLuaSideEffect(); @@ -2027,8 +2163,8 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) nmg.addMask(*str); } else { - for(const auto& p : boost::get>>(inp)) { - nmg.addMask(p.second); + for (const auto& p : boost::get>>(inp)) { + nmg.addMask(p.second); } } g_proxyProtocolACL = std::move(nmg); @@ -2037,7 +2173,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("setProxyProtocolApplyACLToProxiedClients", [](bool apply) { if (g_configurationDone) { errlog("setProxyProtocolApplyACLToProxiedClients() cannot be used at runtime!"); - g_outputBuffer="setProxyProtocolApplyACLToProxiedClients() cannot be used at runtime!\n"; + g_outputBuffer = "setProxyProtocolApplyACLToProxiedClients() cannot be used at runtime!\n"; return; } setLuaSideEffect(); @@ -2047,7 +2183,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("setProxyProtocolMaximumPayloadSize", [](size_t size) { if (g_configurationDone) { errlog("setProxyProtocolMaximumPayloadSize() cannot be used at runtime!"); - g_outputBuffer="setProxyProtocolMaximumPayloadSize() cannot be used at runtime!\n"; + g_outputBuffer = "setProxyProtocolMaximumPayloadSize() cannot be used at runtime!\n"; return; } setLuaSideEffect(); @@ -2055,112 +2191,111 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) }); luaCtx.writeFunction("setUDPMultipleMessagesVectorSize", [](size_t vSize) { - if (g_configurationDone) { - errlog("setUDPMultipleMessagesVectorSize() cannot be used at runtime!"); - g_outputBuffer="setUDPMultipleMessagesVectorSize() cannot be used at runtime!\n"; - return; - } + if (g_configurationDone) { + errlog("setUDPMultipleMessagesVectorSize() cannot be used at runtime!"); + g_outputBuffer = "setUDPMultipleMessagesVectorSize() cannot be used at runtime!\n"; + return; + } #if defined(HAVE_RECVMMSG) && defined(HAVE_SENDMMSG) && defined(MSG_WAITFORONE) - setLuaSideEffect(); - g_udpVectorSize = vSize; + setLuaSideEffect(); + g_udpVectorSize = vSize; #else errlog("recvmmsg() support is not available!"); - g_outputBuffer="recvmmsg support is not available!\n"; + g_outputBuffer = "recvmmsg support is not available!\n"; #endif - }); + }); luaCtx.writeFunction("setAddEDNSToSelfGeneratedResponses", [](bool add) { - g_addEDNSToSelfGeneratedResponses = add; + g_addEDNSToSelfGeneratedResponses = add; }); luaCtx.writeFunction("setPayloadSizeOnSelfGeneratedAnswers", [](uint16_t payloadSize) { - if (payloadSize < 512) { - warnlog("setPayloadSizeOnSelfGeneratedAnswers() is set too low, using 512 instead!"); - g_outputBuffer="setPayloadSizeOnSelfGeneratedAnswers() is set too low, using 512 instead!"; - payloadSize = 512; - } - if (payloadSize > s_udpIncomingBufferSize) { - warnlog("setPayloadSizeOnSelfGeneratedAnswers() is set too high, capping to %d instead!", s_udpIncomingBufferSize); - g_outputBuffer="setPayloadSizeOnSelfGeneratedAnswers() is set too high, capping to " + std::to_string(s_udpIncomingBufferSize) + " instead"; - payloadSize = s_udpIncomingBufferSize; - } - g_PayloadSizeSelfGenAnswers = payloadSize; + if (payloadSize < 512) { + warnlog("setPayloadSizeOnSelfGeneratedAnswers() is set too low, using 512 instead!"); + g_outputBuffer = "setPayloadSizeOnSelfGeneratedAnswers() is set too low, using 512 instead!"; + payloadSize = 512; + } + if (payloadSize > s_udpIncomingBufferSize) { + warnlog("setPayloadSizeOnSelfGeneratedAnswers() is set too high, capping to %d instead!", s_udpIncomingBufferSize); + g_outputBuffer = "setPayloadSizeOnSelfGeneratedAnswers() is set too high, capping to " + std::to_string(s_udpIncomingBufferSize) + " instead"; + payloadSize = s_udpIncomingBufferSize; + } + g_PayloadSizeSelfGenAnswers = payloadSize; }); luaCtx.writeFunction("setSecurityPollSuffix", [](const std::string& suffix) { - if (g_configurationDone) { - g_outputBuffer="setSecurityPollSuffix() cannot be used at runtime!\n"; - return; - } + if (g_configurationDone) { + g_outputBuffer = "setSecurityPollSuffix() cannot be used at runtime!\n"; + return; + } - g_secPollSuffix = suffix; + g_secPollSuffix = suffix; }); luaCtx.writeFunction("setSecurityPollInterval", [](time_t newInterval) { - if (newInterval <= 0) { - warnlog("setSecurityPollInterval() should be > 0, skipping"); - g_outputBuffer="setSecurityPollInterval() should be > 0, skipping"; - } + if (newInterval <= 0) { + warnlog("setSecurityPollInterval() should be > 0, skipping"); + g_outputBuffer = "setSecurityPollInterval() should be > 0, skipping"; + } - g_secPollInterval = newInterval; + g_secPollInterval = newInterval; }); - luaCtx.writeFunction("setSyslogFacility", [](boost::variant facility) { + luaCtx.writeFunction("setSyslogFacility", [](boost::variant facility) { setLuaSideEffect(); if (g_configurationDone) { - g_outputBuffer="setSyslogFacility cannot be used at runtime!\n"; + g_outputBuffer = "setSyslogFacility cannot be used at runtime!\n"; return; } if (facility.type() == typeid(std::string)) { static std::map const facilities = { - { "local0", LOG_LOCAL0 }, - { "log_local0", LOG_LOCAL0 }, - { "local1", LOG_LOCAL1 }, - { "log_local1", LOG_LOCAL1 }, - { "local2", LOG_LOCAL2 }, - { "log_local2", LOG_LOCAL2 }, - { "local3", LOG_LOCAL3 }, - { "log_local3", LOG_LOCAL3 }, - { "local4", LOG_LOCAL4 }, - { "log_local4", LOG_LOCAL4 }, - { "local5", LOG_LOCAL5 }, - { "log_local5", LOG_LOCAL5 }, - { "local6", LOG_LOCAL6 }, - { "log_local6", LOG_LOCAL6 }, - { "local7", LOG_LOCAL7 }, - { "log_local7", LOG_LOCAL7 }, + {"local0", LOG_LOCAL0}, + {"log_local0", LOG_LOCAL0}, + {"local1", LOG_LOCAL1}, + {"log_local1", LOG_LOCAL1}, + {"local2", LOG_LOCAL2}, + {"log_local2", LOG_LOCAL2}, + {"local3", LOG_LOCAL3}, + {"log_local3", LOG_LOCAL3}, + {"local4", LOG_LOCAL4}, + {"log_local4", LOG_LOCAL4}, + {"local5", LOG_LOCAL5}, + {"log_local5", LOG_LOCAL5}, + {"local6", LOG_LOCAL6}, + {"log_local6", LOG_LOCAL6}, + {"local7", LOG_LOCAL7}, + {"log_local7", LOG_LOCAL7}, /* most of these likely make very little sense for dnsdist, but why not? */ - { "kern", LOG_KERN }, - { "log_kern", LOG_KERN }, - { "user", LOG_USER }, - { "log_user", LOG_USER }, - { "mail", LOG_MAIL }, - { "log_mail", LOG_MAIL }, - { "daemon", LOG_DAEMON }, - { "log_daemon", LOG_DAEMON }, - { "auth", LOG_AUTH }, - { "log_auth", LOG_AUTH }, - { "syslog", LOG_SYSLOG }, - { "log_syslog", LOG_SYSLOG }, - { "lpr", LOG_LPR }, - { "log_lpr", LOG_LPR }, - { "news", LOG_NEWS }, - { "log_news", LOG_NEWS }, - { "uucp", LOG_UUCP }, - { "log_uucp", LOG_UUCP }, - { "cron", LOG_CRON }, - { "log_cron", LOG_CRON }, - { "authpriv", LOG_AUTHPRIV }, - { "log_authpriv", LOG_AUTHPRIV }, - { "ftp", LOG_FTP }, - { "log_ftp", LOG_FTP } - }; + {"kern", LOG_KERN}, + {"log_kern", LOG_KERN}, + {"user", LOG_USER}, + {"log_user", LOG_USER}, + {"mail", LOG_MAIL}, + {"log_mail", LOG_MAIL}, + {"daemon", LOG_DAEMON}, + {"log_daemon", LOG_DAEMON}, + {"auth", LOG_AUTH}, + {"log_auth", LOG_AUTH}, + {"syslog", LOG_SYSLOG}, + {"log_syslog", LOG_SYSLOG}, + {"lpr", LOG_LPR}, + {"log_lpr", LOG_LPR}, + {"news", LOG_NEWS}, + {"log_news", LOG_NEWS}, + {"uucp", LOG_UUCP}, + {"log_uucp", LOG_UUCP}, + {"cron", LOG_CRON}, + {"log_cron", LOG_CRON}, + {"authpriv", LOG_AUTHPRIV}, + {"log_authpriv", LOG_AUTHPRIV}, + {"ftp", LOG_FTP}, + {"log_ftp", LOG_FTP}}; auto facilityStr = boost::get(facility); toLowerInPlace(facilityStr); auto it = facilities.find(facilityStr); if (it == facilities.end()) { - g_outputBuffer="Unknown facility '" + facilityStr + "' passed to setSyslogFacility()!\n"; + g_outputBuffer = "Unknown facility '" + facilityStr + "' passed to setSyslogFacility()!\n"; return; } setSyslogFacility(it->second); @@ -2170,14 +2305,14 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) } }); - luaCtx.writeFunction("addDOHLocal", [client](const std::string& addr, boost::optional>>> certFiles, boost::optional>>> keyFiles, boost::optional > > > urls, boost::optional vars) { + luaCtx.writeFunction("addDOHLocal", [client](const std::string& addr, boost::optional>>> certFiles, boost::optional>>> keyFiles, boost::optional>>> urls, boost::optional vars) { if (client) { return; } #ifdef HAVE_DNS_OVER_HTTPS setLuaSideEffect(); if (g_configurationDone) { - g_outputBuffer="addDOHLocal cannot be used at runtime!\n"; + g_outputBuffer = "addDOHLocal cannot be used at runtime!\n"; return; } auto frontend = std::make_shared(); @@ -2198,9 +2333,9 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) if (urls->type() == typeid(std::string)) { frontend->d_urls.push_back(boost::get(*urls)); } - else if (urls->type() == typeid(std::vector>)) { - auto urlsVect = boost::get>>(*urls); - for(const auto& p : urlsVect) { + else if (urls->type() == typeid(std::vector>)) { + auto urlsVect = boost::get>>(*urls); + for (const auto& p : urlsVect) { frontend->d_urls.push_back(p.second); } } @@ -2229,9 +2364,8 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) } if (vars->count("customResponseHeaders")) { - for (auto const& headerMap : boost::get>((*vars)["customResponseHeaders"])) { - std::pair headerResponse = std::make_pair(boost::to_lower_copy(headerMap.first), headerMap.second); - frontend->d_customResponseHeaders.push_back(headerResponse); + for (auto const& headerMap : boost::get>((*vars).at("customResponseHeaders"))) { + frontend->d_customResponseHeaders.emplace_back(boost::to_lower_copy(headerMap.first), headerMap.second); } } @@ -2254,7 +2388,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) parseTLSConfig(frontend->d_tlsConfig, "addDOHLocal", vars); } g_dohlocals.push_back(frontend); - auto cs = std::unique_ptr(new ClientState(frontend->d_local, true, reusePort, tcpFastOpenQueueSize, interface, cpus)); + auto cs = std::make_unique(frontend->d_local, true, reusePort, tcpFastOpenQueueSize, interface, cpus); cs->dohFrontend = frontend; if (tcpListenQueueSize > 0) { cs->tcpListenQueueSize = tcpListenQueueSize; @@ -2264,373 +2398,388 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) } g_frontends.push_back(std::move(cs)); #else - throw std::runtime_error("addDOHLocal() called but DNS over HTTPS support is not present!"); + throw std::runtime_error("addDOHLocal() called but DNS over HTTPS support is not present!"); #endif }); luaCtx.writeFunction("showDOHFrontends", []() { #ifdef HAVE_DNS_OVER_HTTPS - setLuaNoSideEffect(); - try { - ostringstream ret; - boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d"); - ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid" % "# ticket keys" % "Rotation delay" % "Next rotation") << endl; - size_t counter = 0; - for (const auto& ctx : g_dohlocals) { - ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http2Stats.d_nbQueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses % ctx->getTicketsKeysCount() % ctx->getTicketsKeyRotationDelay() % ctx->getNextTicketsKeyRotation()) << endl; - counter++; - } - g_outputBuffer = ret.str(); - } - catch(const std::exception& e) { - g_outputBuffer = e.what(); - throw; - } + setLuaNoSideEffect(); + try { + ostringstream ret; + boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d"); + ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid" % "# ticket keys" % "Rotation delay" % "Next rotation") << endl; + size_t counter = 0; + for (const auto& ctx : g_dohlocals) { + ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http2Stats.d_nbQueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses % ctx->getTicketsKeysCount() % ctx->getTicketsKeyRotationDelay() % ctx->getNextTicketsKeyRotation()) << endl; + counter++; + } + g_outputBuffer = ret.str(); + } + catch (const std::exception& e) { + g_outputBuffer = e.what(); + throw; + } #else - g_outputBuffer="DNS over HTTPS support is not present!\n"; + g_outputBuffer = "DNS over HTTPS support is not present!\n"; #endif - }); + }); - luaCtx.writeFunction("showDOHResponseCodes", []() { + luaCtx.writeFunction("showDOHResponseCodes", []() { #ifdef HAVE_DNS_OVER_HTTPS - setLuaNoSideEffect(); - try { - ostringstream ret; - boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d"); - g_outputBuffer = "\n- HTTP/1:\n\n"; - ret << (fmt % "#" % "Address" % "200" % "400" % "403" % "500" % "502" % "Others" ) << endl; - size_t counter = 0; - for (const auto& ctx : g_dohlocals) { - ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_http1Stats.d_nb200Responses % ctx->d_http1Stats.d_nb400Responses % ctx->d_http1Stats.d_nb403Responses % ctx->d_http1Stats.d_nb500Responses % ctx->d_http1Stats.d_nb502Responses % ctx->d_http1Stats.d_nbOtherResponses) << endl; - counter++; - } - g_outputBuffer += ret.str(); - ret.str(""); - - g_outputBuffer += "\n- HTTP/2:\n\n"; - ret << (fmt % "#" % "Address" % "200" % "400" % "403" % "500" % "502" % "Others" ) << endl; - counter = 0; - for (const auto& ctx : g_dohlocals) { - ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_http2Stats.d_nb200Responses % ctx->d_http2Stats.d_nb400Responses % ctx->d_http2Stats.d_nb403Responses % ctx->d_http2Stats.d_nb500Responses % ctx->d_http2Stats.d_nb502Responses % ctx->d_http2Stats.d_nbOtherResponses) << endl; - counter++; - } - g_outputBuffer += ret.str(); - } - catch(const std::exception& e) { - g_outputBuffer = e.what(); - throw; - } + setLuaNoSideEffect(); + try { + ostringstream ret; + boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d"); + g_outputBuffer = "\n- HTTP/1:\n\n"; + ret << (fmt % "#" % "Address" % "200" % "400" % "403" % "500" % "502" % "Others") << endl; + size_t counter = 0; + for (const auto& ctx : g_dohlocals) { + ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_http1Stats.d_nb200Responses % ctx->d_http1Stats.d_nb400Responses % ctx->d_http1Stats.d_nb403Responses % ctx->d_http1Stats.d_nb500Responses % ctx->d_http1Stats.d_nb502Responses % ctx->d_http1Stats.d_nbOtherResponses) << endl; + counter++; + } + g_outputBuffer += ret.str(); + ret.str(""); + + g_outputBuffer += "\n- HTTP/2:\n\n"; + ret << (fmt % "#" % "Address" % "200" % "400" % "403" % "500" % "502" % "Others") << endl; + counter = 0; + for (const auto& ctx : g_dohlocals) { + ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_http2Stats.d_nb200Responses % ctx->d_http2Stats.d_nb400Responses % ctx->d_http2Stats.d_nb403Responses % ctx->d_http2Stats.d_nb500Responses % ctx->d_http2Stats.d_nb502Responses % ctx->d_http2Stats.d_nbOtherResponses) << endl; + counter++; + } + g_outputBuffer += ret.str(); + } + catch (const std::exception& e) { + g_outputBuffer = e.what(); + throw; + } #else - g_outputBuffer="DNS over HTTPS support is not present!\n"; + g_outputBuffer = "DNS over HTTPS support is not present!\n"; #endif - }); + }); - luaCtx.writeFunction("getDOHFrontend", [client](size_t index) { - std::shared_ptr result = nullptr; - if (client) { - return result; - } + luaCtx.writeFunction("getDOHFrontend", [client](size_t index) { + std::shared_ptr result = nullptr; + if (client) { + return result; + } #ifdef HAVE_DNS_OVER_HTTPS - setLuaNoSideEffect(); - try { - if (index < g_dohlocals.size()) { - result = g_dohlocals.at(index); - } - else { - errlog("Error: trying to get DOH frontend with index %zu but we only have %zu frontend(s)\n", index, g_dohlocals.size()); - g_outputBuffer="Error: trying to get DOH frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_dohlocals.size()) + " frontend(s)\n"; - } - } - catch(const std::exception& e) { - g_outputBuffer="Error while trying to get DOH frontend with index " + std::to_string(index) + ": "+string(e.what())+"\n"; - errlog("Error while trying to get DOH frontend with index %zu: %s\n", index, string(e.what())); - } + setLuaNoSideEffect(); + try { + if (index < g_dohlocals.size()) { + result = g_dohlocals.at(index); + } + else { + errlog("Error: trying to get DOH frontend with index %zu but we only have %zu frontend(s)\n", index, g_dohlocals.size()); + g_outputBuffer = "Error: trying to get DOH frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_dohlocals.size()) + " frontend(s)\n"; + } + } + catch (const std::exception& e) { + g_outputBuffer = "Error while trying to get DOH frontend with index " + std::to_string(index) + ": " + string(e.what()) + "\n"; + errlog("Error while trying to get DOH frontend with index %zu: %s\n", index, string(e.what())); + } #else g_outputBuffer="DNS over HTTPS support is not present!\n"; #endif - return result; - }); + return result; + }); - luaCtx.writeFunction("getDOHFrontendCount", []() { - setLuaNoSideEffect(); - return g_dohlocals.size(); - }); + luaCtx.writeFunction("getDOHFrontendCount", []() { + setLuaNoSideEffect(); + return g_dohlocals.size(); + }); - luaCtx.registerFunction::*)()>("reloadCertificates", [](std::shared_ptr frontend) { - if (frontend != nullptr) { - frontend->reloadCertificates(); - } - }); + luaCtx.registerFunction::*)()>("reloadCertificates", [](std::shared_ptr frontend) { + if (frontend != nullptr) { + frontend->reloadCertificates(); + } + }); - luaCtx.registerFunction::*)(boost::variant>> certFiles, boost::variant>> keyFiles)>("loadNewCertificatesAndKeys", [](std::shared_ptr frontend, boost::variant>> certFiles, boost::variant>> keyFiles) { + luaCtx.registerFunction::*)(boost::variant>> certFiles, boost::variant>> keyFiles)>("loadNewCertificatesAndKeys", [](std::shared_ptr frontend, boost::variant>> certFiles, boost::variant>> keyFiles) { #ifdef HAVE_DNS_OVER_HTTPS - if (frontend != nullptr) { - if (loadTLSCertificateAndKeys("DOHFrontend::loadNewCertificatesAndKeys", frontend->d_tlsConfig.d_certKeyPairs, certFiles, keyFiles)) { - frontend->reloadCertificates(); - } - } + if (frontend != nullptr) { + if (loadTLSCertificateAndKeys("DOHFrontend::loadNewCertificatesAndKeys", frontend->d_tlsConfig.d_certKeyPairs, certFiles, keyFiles)) { + frontend->reloadCertificates(); + } + } #endif - }); + }); - luaCtx.registerFunction::*)()>("rotateTicketsKey", [](std::shared_ptr frontend) { - if (frontend != nullptr) { - frontend->rotateTicketsKey(time(nullptr)); - } - }); + luaCtx.registerFunction::*)()>("rotateTicketsKey", [](std::shared_ptr frontend) { + if (frontend != nullptr) { + frontend->rotateTicketsKey(time(nullptr)); + } + }); - luaCtx.registerFunction::*)(const std::string&)>("loadTicketsKeys", [](std::shared_ptr frontend, const std::string& file) { - if (frontend != nullptr) { - frontend->loadTicketsKeys(file); - } - }); + luaCtx.registerFunction::*)(const std::string&)>("loadTicketsKeys", [](std::shared_ptr frontend, const std::string& file) { + if (frontend != nullptr) { + frontend->loadTicketsKeys(file); + } + }); - luaCtx.registerFunction::*)(const std::map>&)>("setResponsesMap", [](std::shared_ptr frontend, const std::map>& map) { - if (frontend != nullptr) { - auto newMap = std::make_shared>>(); - newMap->reserve(map.size()); + luaCtx.registerFunction::*)(const std::map>&)>("setResponsesMap", [](std::shared_ptr frontend, const std::map>& map) { + if (frontend != nullptr) { + auto newMap = std::make_shared>>(); + newMap->reserve(map.size()); - for (const auto& entry : map) { - newMap->push_back(entry.second); - } + for (const auto& entry : map) { + newMap->push_back(entry.second); + } - frontend->d_responsesMap = std::move(newMap); - } - }); + frontend->d_responsesMap = std::move(newMap); + } + }); - luaCtx.writeFunction("addTLSLocal", [client](const std::string& addr, boost::variant>> certFiles, boost::variant>> keyFiles, boost::optional vars) { - if (client) { - return; - } + luaCtx.writeFunction("addTLSLocal", [client](const std::string& addr, boost::variant>> certFiles, boost::variant>> keyFiles, boost::optional vars) { + if (client) { + return; + } #ifdef HAVE_DNS_OVER_TLS - setLuaSideEffect(); - if (g_configurationDone) { - g_outputBuffer="addTLSLocal cannot be used at runtime!\n"; - return; - } - shared_ptr frontend = std::make_shared(); + setLuaSideEffect(); + if (g_configurationDone) { + g_outputBuffer = "addTLSLocal cannot be used at runtime!\n"; + return; + } + shared_ptr frontend = std::make_shared(); - if (!loadTLSCertificateAndKeys("addTLSLocal", frontend->d_tlsConfig.d_certKeyPairs, certFiles, keyFiles)) { - return; - } + if (!loadTLSCertificateAndKeys("addTLSLocal", frontend->d_tlsConfig.d_certKeyPairs, certFiles, keyFiles)) { + return; + } - bool reusePort = false; - int tcpFastOpenQueueSize = 0; - int tcpListenQueueSize = 0; - size_t maxInFlightQueriesPerConn = 0; - size_t tcpMaxConcurrentConns = 0; - std::string interface; - std::set cpus; + bool reusePort = false; + int tcpFastOpenQueueSize = 0; + int tcpListenQueueSize = 0; + size_t maxInFlightQueriesPerConn = 0; + size_t tcpMaxConcurrentConns = 0; + std::string interface; + std::set cpus; - if (vars) { - parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConns); + if (vars) { + parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConns); - if (vars->count("provider")) { - frontend->d_provider = boost::get((*vars)["provider"]); - boost::algorithm::to_lower(frontend->d_provider); - } + if (vars->count("provider")) { + frontend->d_provider = boost::get((*vars)["provider"]); + boost::algorithm::to_lower(frontend->d_provider); + } - parseTLSConfig(frontend->d_tlsConfig, "addTLSLocal", vars); - } + parseTLSConfig(frontend->d_tlsConfig, "addTLSLocal", vars); + } - try { - frontend->d_addr = ComboAddress(addr, 853); - if (!frontend->d_provider.empty()) { - vinfolog("Loading TLS provider '%s'", frontend->d_provider); - } - else { + try { + frontend->d_addr = ComboAddress(addr, 853); + if (!frontend->d_provider.empty()) { + vinfolog("Loading TLS provider '%s'", frontend->d_provider); + } + else { #ifdef HAVE_LIBSSL - vinfolog("Loading default TLS provider 'openssl'"); + vinfolog("Loading default TLS provider 'openssl'"); #else - vinfolog("Loading default TLS provider 'gnutls'"); + vinfolog("Loading default TLS provider 'gnutls'"); #endif - } - // only works pre-startup, so no sync necessary - auto cs = std::unique_ptr(new ClientState(frontend->d_addr, true, reusePort, tcpFastOpenQueueSize, interface, cpus)); - cs->tlsFrontend = frontend; - if (tcpListenQueueSize > 0) { - cs->tcpListenQueueSize = tcpListenQueueSize; - } - if (maxInFlightQueriesPerConn > 0) { - cs->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; - } - if (tcpMaxConcurrentConns > 0) { - cs->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConns; - } + } + // only works pre-startup, so no sync necessary + auto cs = std::make_unique(frontend->d_addr, true, reusePort, tcpFastOpenQueueSize, interface, cpus); + cs->tlsFrontend = frontend; + if (tcpListenQueueSize > 0) { + cs->tcpListenQueueSize = tcpListenQueueSize; + } + if (maxInFlightQueriesPerConn > 0) { + cs->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn; + } + if (tcpMaxConcurrentConns > 0) { + cs->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConns; + } - g_tlslocals.push_back(cs->tlsFrontend); - g_frontends.push_back(std::move(cs)); - } - catch(const std::exception& e) { - g_outputBuffer="Error: "+string(e.what())+"\n"; - } + g_tlslocals.push_back(cs->tlsFrontend); + g_frontends.push_back(std::move(cs)); + } + catch (const std::exception& e) { + g_outputBuffer = "Error: " + string(e.what()) + "\n"; + } #else - throw std::runtime_error("addTLSLocal() called but DNS over TLS support is not present!"); + throw std::runtime_error("addTLSLocal() called but DNS over TLS support is not present!"); #endif - }); + }); - luaCtx.writeFunction("showTLSContexts", []() { + luaCtx.writeFunction("showTLSContexts", []() { #ifdef HAVE_DNS_OVER_TLS - setLuaNoSideEffect(); - try { - ostringstream ret; - boost::format fmt("%1$-3d %2$-20.20s %|25t|%3$-14d %|40t|%4$-14d %|54t|%5$-21.21s"); - // 1 2 3 4 5 - ret << (fmt % "#" % "Address" % "# ticket keys" % "Rotation delay" % "Next rotation" ) << endl; - size_t counter = 0; - for (const auto& ctx : g_tlslocals) { - ret << (fmt % counter % ctx->d_addr.toStringWithPort() % ctx->getTicketsKeysCount() % ctx->getTicketsKeyRotationDelay() % ctx->getNextTicketsKeyRotation()) << endl; - counter++; - } - g_outputBuffer = ret.str(); - } - catch(const std::exception& e) { - g_outputBuffer = e.what(); - throw; - } + setLuaNoSideEffect(); + try { + ostringstream ret; + boost::format fmt("%1$-3d %2$-20.20s %|25t|%3$-14d %|40t|%4$-14d %|54t|%5$-21.21s"); + // 1 2 3 4 5 + ret << (fmt % "#" % "Address" % "# ticket keys" % "Rotation delay" % "Next rotation") << endl; + size_t counter = 0; + for (const auto& ctx : g_tlslocals) { + ret << (fmt % counter % ctx->d_addr.toStringWithPort() % ctx->getTicketsKeysCount() % ctx->getTicketsKeyRotationDelay() % ctx->getNextTicketsKeyRotation()) << endl; + counter++; + } + g_outputBuffer = ret.str(); + } + catch (const std::exception& e) { + g_outputBuffer = e.what(); + throw; + } #else - g_outputBuffer="DNS over TLS support is not present!\n"; + g_outputBuffer = "DNS over TLS support is not present!\n"; #endif - }); + }); - luaCtx.writeFunction("getTLSContext", [](size_t index) { - std::shared_ptr result = nullptr; + luaCtx.writeFunction("getTLSContext", [](size_t index) { + std::shared_ptr result = nullptr; #ifdef HAVE_DNS_OVER_TLS - setLuaNoSideEffect(); - try { - if (index < g_tlslocals.size()) { - result = g_tlslocals.at(index)->getContext(); - } - else { - errlog("Error: trying to get TLS context with index %zu but we only have %zu context(s)\n", index, g_tlslocals.size()); - g_outputBuffer="Error: trying to get TLS context with index " + std::to_string(index) + " but we only have " + std::to_string(g_tlslocals.size()) + " context(s)\n"; - } - } - catch(const std::exception& e) { - g_outputBuffer="Error while trying to get TLS context with index " + std::to_string(index) + ": "+string(e.what())+"\n"; - errlog("Error while trying to get TLS context with index %zu: %s\n", index, string(e.what())); - } + setLuaNoSideEffect(); + try { + if (index < g_tlslocals.size()) { + result = g_tlslocals.at(index)->getContext(); + } + else { + errlog("Error: trying to get TLS context with index %zu but we only have %zu context(s)\n", index, g_tlslocals.size()); + g_outputBuffer = "Error: trying to get TLS context with index " + std::to_string(index) + " but we only have " + std::to_string(g_tlslocals.size()) + " context(s)\n"; + } + } + catch (const std::exception& e) { + g_outputBuffer = "Error while trying to get TLS context with index " + std::to_string(index) + ": " + string(e.what()) + "\n"; + errlog("Error while trying to get TLS context with index %zu: %s\n", index, string(e.what())); + } #else g_outputBuffer="DNS over TLS support is not present!\n"; #endif - return result; - }); + return result; + }); - luaCtx.writeFunction("getTLSFrontend", [](size_t index) { - std::shared_ptr result = nullptr; + luaCtx.writeFunction("getTLSFrontend", [](size_t index) { + std::shared_ptr result = nullptr; #ifdef HAVE_DNS_OVER_TLS - setLuaNoSideEffect(); - try { - if (index < g_tlslocals.size()) { - result = g_tlslocals.at(index); - } - else { - errlog("Error: trying to get TLS frontend with index %zu but we only have %zu frontends\n", index, g_tlslocals.size()); - g_outputBuffer="Error: trying to get TLS frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_tlslocals.size()) + " frontend(s)\n"; - } - } - catch(const std::exception& e) { - g_outputBuffer="Error while trying to get TLS frontend with index " + std::to_string(index) + ": "+string(e.what())+"\n"; - errlog("Error while trying to get TLS frontend with index %zu: %s\n", index, string(e.what())); - } + setLuaNoSideEffect(); + try { + if (index < g_tlslocals.size()) { + result = g_tlslocals.at(index); + } + else { + errlog("Error: trying to get TLS frontend with index %zu but we only have %zu frontends\n", index, g_tlslocals.size()); + g_outputBuffer = "Error: trying to get TLS frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_tlslocals.size()) + " frontend(s)\n"; + } + } + catch (const std::exception& e) { + g_outputBuffer = "Error while trying to get TLS frontend with index " + std::to_string(index) + ": " + string(e.what()) + "\n"; + errlog("Error while trying to get TLS frontend with index %zu: %s\n", index, string(e.what())); + } #else g_outputBuffer="DNS over TLS support is not present!\n"; #endif - return result; - }); + return result; + }); - luaCtx.writeFunction("getTLSFrontendCount", []() { - setLuaNoSideEffect(); - return g_tlslocals.size(); - }); + luaCtx.writeFunction("getTLSFrontendCount", []() { + setLuaNoSideEffect(); + return g_tlslocals.size(); + }); - luaCtx.registerFunction::*)()>("rotateTicketsKey", [](std::shared_ptr& ctx) { - if (ctx != nullptr) { - ctx->rotateTicketsKey(time(nullptr)); - } - }); + luaCtx.registerFunction::*)()>("rotateTicketsKey", [](std::shared_ptr& ctx) { + if (ctx != nullptr) { + ctx->rotateTicketsKey(time(nullptr)); + } + }); - luaCtx.registerFunction::*)(const std::string&)>("loadTicketsKeys", [](std::shared_ptr& ctx, const std::string& file) { - if (ctx != nullptr) { - ctx->loadTicketsKeys(file); - } - }); + luaCtx.registerFunction::*)(const std::string&)>("loadTicketsKeys", [](std::shared_ptr& ctx, const std::string& file) { + if (ctx != nullptr) { + ctx->loadTicketsKeys(file); + } + }); - luaCtx.registerFunction::*)()>("rotateTicketsKey", [](std::shared_ptr& frontend) { - if (frontend == nullptr) { - return; - } - auto ctx = frontend->getContext(); - if (ctx) { - ctx->rotateTicketsKey(time(nullptr)); - } - }); + luaCtx.registerFunction::*)()>("rotateTicketsKey", [](std::shared_ptr& frontend) { + if (frontend == nullptr) { + return; + } + auto ctx = frontend->getContext(); + if (ctx) { + ctx->rotateTicketsKey(time(nullptr)); + } + }); - luaCtx.registerFunction::*)(const std::string&)>("loadTicketsKeys", [](std::shared_ptr& frontend, const std::string& file) { - if (frontend == nullptr) { - return; - } - auto ctx = frontend->getContext(); - if (ctx) { - ctx->loadTicketsKeys(file); - } - }); + luaCtx.registerFunction::*)(const std::string&)>("loadTicketsKeys", [](std::shared_ptr& frontend, const std::string& file) { + if (frontend == nullptr) { + return; + } + auto ctx = frontend->getContext(); + if (ctx) { + ctx->loadTicketsKeys(file); + } + }); - luaCtx.registerFunction::*)()>("reloadCertificates", [](std::shared_ptr& frontend) { - if (frontend == nullptr) { - return; - } - frontend->setupTLS(); - }); + luaCtx.registerFunction::*)()>("reloadCertificates", [](std::shared_ptr& frontend) { + if (frontend == nullptr) { + return; + } + frontend->setupTLS(); + }); - luaCtx.registerFunction::*)(boost::variant>> certFiles, boost::variant>> keyFiles)>("loadNewCertificatesAndKeys", [](std::shared_ptr& frontend, boost::variant>> certFiles, boost::variant>> keyFiles) { + luaCtx.registerFunction::*)(boost::variant>> certFiles, boost::variant>> keyFiles)>("loadNewCertificatesAndKeys", [](std::shared_ptr& frontend, boost::variant>> certFiles, boost::variant>> keyFiles) { #ifdef HAVE_DNS_OVER_TLS - if (loadTLSCertificateAndKeys("TLSFrontend::loadNewCertificatesAndKeys", frontend->d_tlsConfig.d_certKeyPairs, certFiles, keyFiles)) { - frontend->setupTLS(); - } + if (loadTLSCertificateAndKeys("TLSFrontend::loadNewCertificatesAndKeys", frontend->d_tlsConfig.d_certKeyPairs, certFiles, keyFiles)) { + frontend->setupTLS(); + } #endif - }); + }); - luaCtx.writeFunction("reloadAllCertificates", []() { - for (auto& frontend : g_frontends) { - if (!frontend) { - continue; - } - try { + luaCtx.writeFunction("reloadAllCertificates", []() { + for (auto& frontend : g_frontends) { + if (!frontend) { + continue; + } + try { #ifdef HAVE_DNSCRYPT - if (frontend->dnscryptCtx) { - frontend->dnscryptCtx->reloadCertificates(); - } + if (frontend->dnscryptCtx) { + frontend->dnscryptCtx->reloadCertificates(); + } #endif /* HAVE_DNSCRYPT */ #ifdef HAVE_DNS_OVER_TLS - if (frontend->tlsFrontend) { - frontend->tlsFrontend->setupTLS(); - } + if (frontend->tlsFrontend) { + frontend->tlsFrontend->setupTLS(); + } #endif /* HAVE_DNS_OVER_TLS */ #ifdef HAVE_DNS_OVER_HTTPS - if (frontend->dohFrontend) { - frontend->dohFrontend->reloadCertificates(); - } -#endif /* HAVE_DNS_OVER_HTTPS */ - } - catch(const std::exception& e) { - errlog("Error reloading certificates for frontend %s: %s", frontend->local.toStringWithPort(), e.what()); - } + if (frontend->dohFrontend) { + frontend->dohFrontend->reloadCertificates(); } - }); +#endif /* HAVE_DNS_OVER_HTTPS */ + } + catch (const std::exception& e) { + errlog("Error reloading certificates for frontend %s: %s", frontend->local.toStringWithPort(), e.what()); + } + } + }); - luaCtx.writeFunction("setAllowEmptyResponse", [](bool allow) { g_allowEmptyResponse=allow; }); - luaCtx.writeFunction("setDropEmptyQueries", [](bool drop) { extern bool g_dropEmptyQueries; g_dropEmptyQueries = drop; }); + luaCtx.writeFunction("setAllowEmptyResponse", [](bool allow) { g_allowEmptyResponse = allow; }); + luaCtx.writeFunction("setDropEmptyQueries", [](bool drop) { extern bool g_dropEmptyQueries; g_dropEmptyQueries = drop; }); #if defined(HAVE_LIBSSL) && defined(HAVE_OCSP_BASIC_SIGN) - luaCtx.writeFunction("generateOCSPResponse", [client](const std::string& certFile, const std::string& caCert, const std::string& caKey, const std::string& outFile, int ndays, int nmin) { - if (client) { - return; - } + luaCtx.writeFunction("generateOCSPResponse", [client](const std::string& certFile, const std::string& caCert, const std::string& caKey, const std::string& outFile, int ndays, int nmin) { + if (client) { + return; + } - libssl_generate_ocsp_response(certFile, caCert, caKey, outFile, ndays, nmin); - }); + libssl_generate_ocsp_response(certFile, caCert, caKey, outFile, ndays, nmin); + }); #endif /* HAVE_LIBSSL && HAVE_OCSP_BASIC_SIGN*/ + + luaCtx.writeFunction("addCapabilitiesToRetain", [](boost::variant> caps) { + setLuaSideEffect(); + if (g_configurationDone) { + g_outputBuffer = "addCapabilitiesToRetain cannot be used at runtime!\n"; + return; + } + if (caps.type() == typeid(std::string)) { + g_capabilitiesToRetain.insert(boost::get(caps)); + } + else if (caps.type() == typeid(std::map)) { + for (const auto& cap : boost::get>(caps)) { + g_capabilitiesToRetain.insert(cap.second); + } + } + }); } vector> setupLua(LuaContext& luaCtx, bool client, bool configCheck, const std::string& config) @@ -2657,7 +2806,7 @@ vector> setupLua(LuaContext& luaCtx, bool client, bool #endif std::ifstream ifs(config); - if(!ifs) + if (!ifs) warnlog("Unable to read configuration from '%s'", config); else vinfolog("Read configuration from '%s'", config); diff --git a/pdns/dnsdist-lua.hh b/pdns/dnsdist-lua.hh index 28cd00da7396..b8230ebad60b 100644 --- a/pdns/dnsdist-lua.hh +++ b/pdns/dnsdist-lua.hh @@ -92,7 +92,7 @@ std::shared_ptr makeRule(const luadnsrule_t& var); typedef std::unordered_map > luaruleparams_t; void parseRuleParams(boost::optional params, boost::uuids::uuid& uuid, std::string& name, uint64_t& creationOrder); -typedef NetmaskTree nmts_t; +typedef NetmaskTree nmts_t; vector> setupLua(LuaContext& luaCtx, bool client, bool configCheck, const std::string& config); void setupLuaActions(LuaContext& luaCtx); diff --git a/pdns/dnsdist-protobuf.cc b/pdns/dnsdist-protobuf.cc index e2e6c04106d5..df8cb3f6b20d 100644 --- a/pdns/dnsdist-protobuf.cc +++ b/pdns/dnsdist-protobuf.cc @@ -78,12 +78,12 @@ void DNSDistProtoBufMessage::setBytes(size_t bytes) void DNSDistProtoBufMessage::setTime(time_t sec, uint32_t usec) { - d_time = std::make_pair(sec, usec); + d_time = std::pair(sec, usec); } void DNSDistProtoBufMessage::setQueryTime(time_t sec, uint32_t usec) { - d_queryTime = std::make_pair(sec, usec); + d_queryTime = std::pair(sec, usec); } void DNSDistProtoBufMessage::setQuestion(const DNSName& name, uint16_t qtype, uint16_t qclass) @@ -124,7 +124,26 @@ void DNSDistProtoBufMessage::serialize(std::string& data) const m.setTime(ts.tv_sec, ts.tv_nsec / 1000); } - m.setRequest(d_dq.uniqueId ? *d_dq.uniqueId : getUniqueID(), d_requestor ? *d_requestor : *d_dq.remote, d_responder ? *d_responder : *d_dq.local, d_question ? d_question->d_name : *d_dq.qname, d_question ? d_question->d_type : d_dq.qtype, d_question ? d_question->d_class : d_dq.qclass, d_dq.getHeader()->id, (d_dq.getProtocol() == dnsdist::Protocol::DoH) ? true : d_dq.overTCP(), d_bytes ? *d_bytes : d_dq.getData().size()); + const auto distProto = d_dq.getProtocol(); + pdns::ProtoZero::Message::TransportProtocol protocol = pdns::ProtoZero::Message::TransportProtocol::UDP; + + if (distProto == dnsdist::Protocol::DoTCP) { + protocol = pdns::ProtoZero::Message::TransportProtocol::TCP; + } + else if (distProto == dnsdist::Protocol::DoT) { + protocol = pdns::ProtoZero::Message::TransportProtocol::DoT; + } + else if (distProto == dnsdist::Protocol::DoH) { + protocol = pdns::ProtoZero::Message::TransportProtocol::DoH; + } + else if (distProto == dnsdist::Protocol::DNSCryptUDP) { + protocol = pdns::ProtoZero::Message::TransportProtocol::DNSCryptUDP; + } + else if (distProto == dnsdist::Protocol::DNSCryptTCP) { + protocol = pdns::ProtoZero::Message::TransportProtocol::DNSCryptTCP; + } + + m.setRequest(d_dq.uniqueId ? *d_dq.uniqueId : getUniqueID(), d_requestor ? *d_requestor : *d_dq.remote, d_responder ? *d_responder : *d_dq.local, d_question ? d_question->d_name : *d_dq.qname, d_question ? d_question->d_type : d_dq.qtype, d_question ? d_question->d_class : d_dq.qclass, d_dq.getHeader()->id, protocol, d_bytes ? *d_bytes : d_dq.getData().size()); if (d_serverIdentity) { m.setServerIdentity(*d_serverIdentity); diff --git a/pdns/dnsdist-protocols.cc b/pdns/dnsdist-protocols.cc new file mode 100644 index 000000000000..89b65ecac53c --- /dev/null +++ b/pdns/dnsdist-protocols.cc @@ -0,0 +1,80 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include + +#include "dnsdist-protocols.hh" + +namespace dnsdist +{ +static const std::vector names = { + "DoUDP", + "DoTCP", + "DNSCryptUDP", + "DNSCryptTCP", + "DoT", + "DoH"}; + +static const std::vector prettyNames = { + "Do53 UDP", + "Do53 TCP", + "DNSCrypt UDP", + "DNSCrypt TCP", + "DNS over TLS", + "DNS over HTTPS"}; + +Protocol::Protocol(Protocol::typeenum protocol) : + d_protocol(protocol) +{ + if (protocol >= names.size()) { + throw std::runtime_error("Unknown protocol: '" + std::to_string(protocol) + "'"); + } +} + +Protocol::Protocol(const std::string& s) +{ + const auto& it = std::find(names.begin(), names.end(), s); + if (it == names.end()) { + throw std::runtime_error("Unknown protocol name: '" + s + "'"); + } + + auto index = std::distance(names.begin(), it); + d_protocol = static_cast(index); +} + +bool Protocol::operator==(Protocol::typeenum type) const +{ + return d_protocol == type; +} + +const std::string& Protocol::toString() const +{ + return names.at(static_cast(d_protocol)); +} + +const std::string& Protocol::toPrettyString() const +{ + return prettyNames.at(static_cast(d_protocol)); +} + +} diff --git a/pdns/dnsdist-protocols.hh b/pdns/dnsdist-protocols.hh index 0b3b540a0a25..a90e6e0c5602 100644 --- a/pdns/dnsdist-protocols.hh +++ b/pdns/dnsdist-protocols.hh @@ -26,15 +26,28 @@ namespace dnsdist { -enum class Protocol : uint8_t +class Protocol { - DoUDP, - DoTCP, - DNSCryptUDP, - DNSCryptTCP, - DoT, - DoH -}; +public: + enum typeenum : uint8_t + { + DoUDP, + DoTCP, + DNSCryptUDP, + DNSCryptTCP, + DoT, + DoH + }; + + Protocol(typeenum protocol = DoUDP); + explicit Protocol(const std::string& protocol); + + bool operator==(typeenum) const; -const std::string& ProtocolToString(Protocol proto); + const std::string& toString() const; + const std::string& toPrettyString() const; + +private: + typeenum d_protocol; +}; } diff --git a/pdns/dnsdist-rings.cc b/pdns/dnsdist-rings.cc index ce24a5f646ae..f0c3caacac46 100644 --- a/pdns/dnsdist-rings.cc +++ b/pdns/dnsdist-rings.cc @@ -108,9 +108,9 @@ size_t Rings::loadFromFile(const std::string& filepath, const struct timespec& n vector parts; stringtok(parts, line, " \t,"); - if (parts.size() == 7) { + if (parts.size() == 8) { } - else if (parts.size() >= 10 && parts.size() <= 12) { + else if (parts.size() >= 11 && parts.size() <= 13) { isResponse = true; } else { @@ -138,7 +138,7 @@ size_t Rings::loadFromFile(const std::string& filepath, const struct timespec& n ComboAddress from(parts.at(idx++)); ComboAddress to; - + dnsdist::Protocol protocol(parts.at(idx++)); if (isResponse) { to = ComboAddress(parts.at(idx++)); } @@ -148,10 +148,10 @@ size_t Rings::loadFromFile(const std::string& filepath, const struct timespec& n QType qtype(QType::chartocode(parts.at(idx++).c_str())); if (isResponse) { - insertResponse(when, from, qname, qtype.getCode(), 0, 0, dh, to); + insertResponse(when, from, qname, qtype.getCode(), 0, 0, dh, to, protocol); } else { - insertQuery(when, from, qname, qtype.getCode(), 0, dh); + insertQuery(when, from, qname, qtype.getCode(), 0, dh, protocol); } ++inserted; } diff --git a/pdns/dnsdist-rings.hh b/pdns/dnsdist-rings.hh index bc424a35c089..0ee4f22be90c 100644 --- a/pdns/dnsdist-rings.hh +++ b/pdns/dnsdist-rings.hh @@ -31,7 +31,7 @@ #include "iputils.hh" #include "lock.hh" #include "stat_t.hh" - +#include "dnsdist-protocols.hh" struct Rings { struct Query @@ -42,6 +42,8 @@ struct Rings { struct dnsheader dh; uint16_t size; uint16_t qtype; + // incoming protocol + dnsdist::Protocol protocol; }; struct Response { @@ -53,6 +55,8 @@ struct Rings { unsigned int usec; unsigned int size; uint16_t qtype; + // outgoing protocol + dnsdist::Protocol protocol; }; struct Shard @@ -80,7 +84,7 @@ struct Rings { /* resize all the rings */ for (auto& shard : d_shards) { - shard = std::unique_ptr(new Shard()); + shard = std::make_unique(); shard->queryRing.lock()->set_capacity(newCapacity / numberOfShards); shard->respRing.lock()->set_capacity(newCapacity / numberOfShards); } @@ -114,50 +118,50 @@ struct Rings { return d_nbResponseEntries; } - void insertQuery(const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, uint16_t size, const struct dnsheader& dh) + void insertQuery(const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, uint16_t size, const struct dnsheader& dh, dnsdist::Protocol protocol) { for (size_t idx = 0; idx < d_nbLockTries; idx++) { auto& shard = getOneShard(); auto lock = shard->queryRing.try_lock(); if (lock.owns_lock()) { - insertQueryLocked(*lock, when, requestor, name, qtype, size, dh); + insertQueryLocked(*lock, when, requestor, name, qtype, size, dh, protocol); return; } if (d_keepLockingStats) { - d_deferredQueryInserts++; + ++d_deferredQueryInserts; } } /* out of luck, let's just wait */ if (d_keepLockingStats) { - d_blockingResponseInserts++; + ++d_blockingResponseInserts; } auto& shard = getOneShard(); auto lock = shard->queryRing.lock(); - insertQueryLocked(*lock, when, requestor, name, qtype, size, dh); + insertQueryLocked(*lock, when, requestor, name, qtype, size, dh, protocol); } - void insertResponse(const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, unsigned int usec, unsigned int size, const struct dnsheader& dh, const ComboAddress& backend) + void insertResponse(const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, unsigned int usec, unsigned int size, const struct dnsheader& dh, const ComboAddress& backend, dnsdist::Protocol protocol) { for (size_t idx = 0; idx < d_nbLockTries; idx++) { auto& shard = getOneShard(); auto lock = shard->respRing.try_lock(); if (lock.owns_lock()) { - insertResponseLocked(*lock, when, requestor, name, qtype, usec, size, dh, backend); + insertResponseLocked(*lock, when, requestor, name, qtype, usec, size, dh, backend, protocol); return; } if (d_keepLockingStats) { - d_deferredResponseInserts++; + ++d_deferredResponseInserts; } } /* out of luck, let's just wait */ if (d_keepLockingStats) { - d_blockingResponseInserts++; + ++d_blockingResponseInserts; } auto& shard = getOneShard(); auto lock = shard->respRing.lock(); - insertResponseLocked(*lock, when, requestor, name, qtype, usec, size, dh, backend); + insertResponseLocked(*lock, when, requestor, name, qtype, usec, size, dh, backend, protocol); } void clear() @@ -197,20 +201,20 @@ private: return d_shards[getShardId()]; } - void insertQueryLocked(boost::circular_buffer& ring, const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, uint16_t size, const struct dnsheader& dh) + void insertQueryLocked(boost::circular_buffer& ring, const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, uint16_t size, const struct dnsheader& dh, dnsdist::Protocol protocol) { if (!ring.full()) { d_nbQueryEntries++; } - ring.push_back({requestor, name, when, dh, size, qtype}); + ring.push_back({requestor, name, when, dh, size, qtype, protocol}); } - void insertResponseLocked(boost::circular_buffer& ring, const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, unsigned int usec, unsigned int size, const struct dnsheader& dh, const ComboAddress& backend) + void insertResponseLocked(boost::circular_buffer& ring, const struct timespec& when, const ComboAddress& requestor, const DNSName& name, uint16_t qtype, unsigned int usec, unsigned int size, const struct dnsheader& dh, const ComboAddress& backend, dnsdist::Protocol protocol) { if (!ring.full()) { d_nbResponseEntries++; } - ring.push_back({requestor, backend, name, when, dh, usec, size, qtype}); + ring.push_back({requestor, backend, name, when, dh, usec, size, qtype, protocol}); } std::atomic d_nbQueryEntries; diff --git a/pdns/dnsdist-snmp.cc b/pdns/dnsdist-snmp.cc index 0456f0c5f7ff..4da43eed866b 100644 --- a/pdns/dnsdist-snmp.cc +++ b/pdns/dnsdist-snmp.cc @@ -468,14 +468,14 @@ bool DNSDistSNMPAgent::sendDNSTrap(const DNSQuestion& dq, const std::string& rea socketFamilyOID, OID_LENGTH(socketFamilyOID), ASN_INTEGER, - (u_char *) &socketFamily, + reinterpret_cast(&socketFamily), sizeof(socketFamily)); snmp_varlist_add_variable(&varList, socketProtocolOID, OID_LENGTH(socketProtocolOID), ASN_INTEGER, - (u_char *) &socketProtocol, + reinterpret_cast(&socketProtocol), sizeof(socketProtocol)); snmp_varlist_add_variable(&varList, @@ -496,21 +496,21 @@ bool DNSDistSNMPAgent::sendDNSTrap(const DNSQuestion& dq, const std::string& rea queryTypeOID, OID_LENGTH(queryTypeOID), ASN_INTEGER, - (u_char *) &queryType, + reinterpret_cast(&queryType), sizeof(queryType)); snmp_varlist_add_variable(&varList, querySizeOID, OID_LENGTH(querySizeOID), ASN_UNSIGNED, - (u_char *) &querySize, + reinterpret_cast(&querySize), sizeof(querySize)); snmp_varlist_add_variable(&varList, queryIDOID, OID_LENGTH(queryIDOID), ASN_UNSIGNED, - (u_char *) &queryID, + reinterpret_cast(&queryID), sizeof(queryID)); snmp_varlist_add_variable(&varList, @@ -524,14 +524,14 @@ bool DNSDistSNMPAgent::sendDNSTrap(const DNSQuestion& dq, const std::string& rea qClassOID, OID_LENGTH(qClassOID), ASN_UNSIGNED, - (u_char *) &qClass, + reinterpret_cast(&qClass), sizeof(qClass)); snmp_varlist_add_variable(&varList, qTypeOID, OID_LENGTH(qTypeOID), ASN_UNSIGNED, - (u_char *) &qType, + reinterpret_cast(&qType), sizeof(qType)); snmp_varlist_add_variable(&varList, diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc index 7673eaacc697..5d3a422a8e93 100644 --- a/pdns/dnsdist-tcp.cc +++ b/pdns/dnsdist-tcp.cc @@ -106,28 +106,33 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState() size_t IncomingTCPConnectionState::clearAllDownstreamConnections() { - return DownstreamConnectionsManager::clear(); + return t_downstreamTCPConnectionsManager.clear(); } std::shared_ptr IncomingTCPConnectionState::getDownstreamConnection(std::shared_ptr& ds, const std::unique_ptr>& tlvs, const struct timeval& now) { std::shared_ptr downstream{nullptr}; - downstream = getActiveDownstreamConnection(ds, tlvs); + downstream = getOwnedDownstreamConnection(ds, tlvs); if (!downstream) { - /* we don't have a connection to this backend active yet, let's get one (it might not be a fresh one, though) */ - downstream = DownstreamConnectionsManager::getConnectionToDownstream(d_threadData.mplexer, ds, now); - registerActiveDownstreamConnection(downstream); + /* we don't have a connection to this backend owned yet, let's get one (it might not be a fresh one, though) */ + downstream = t_downstreamTCPConnectionsManager.getConnectionToDownstream(d_threadData.mplexer, ds, now, std::string()); + if (ds->useProxyProtocol) { + registerOwnedDownstreamConnection(downstream); + } } return downstream; } -static void tcpClientThread(int pipefd, int crossProtocolPipeFD); +static void tcpClientThread(int pipefd, int crossProtocolQueriesPipeFD, int crossProtocolResponsesListenPipeFD, int crossProtocolResponsesWritePipeFD); TCPClientCollection::TCPClientCollection(size_t maxThreads): d_tcpclientthreads(maxThreads), d_maxthreads(maxThreads) { + for (size_t idx = 0; idx < maxThreads; idx++) { + addTCPClientThread(); + } } void TCPClientCollection::addTCPClientThread() @@ -166,18 +171,25 @@ void TCPClientCollection::addTCPClientThread() return; } - int crossProtocolFDs[2] = { -1, -1}; - if (!preparePipe(crossProtocolFDs, "cross-protocol")) { + int crossProtocolQueriesFDs[2] = { -1, -1}; + if (!preparePipe(crossProtocolQueriesFDs, "cross-protocol queries")) { + return; + } + + int crossProtocolResponsesFDs[2] = { -1, -1}; + if (!preparePipe(crossProtocolResponsesFDs, "cross-protocol responses")) { return; } vinfolog("Adding TCP Client thread"); { - std::lock_guard lock(d_mutex); - if (d_numthreads >= d_tcpclientthreads.size()) { vinfolog("Adding a new TCP client thread would exceed the vector size (%d/%d), skipping. Consider increasing the maximum amount of TCP client threads with setMaxTCPClientThreads() in the configuration.", d_numthreads.load(), d_tcpclientthreads.size()); + close(crossProtocolQueriesFDs[0]); + close(crossProtocolQueriesFDs[1]); + close(crossProtocolResponsesFDs[0]); + close(crossProtocolResponsesFDs[1]); close(pipefds[0]); close(pipefds[1]); return; @@ -185,15 +197,17 @@ void TCPClientCollection::addTCPClientThread() /* from now on this side of the pipe will be managed by that object, no need to worry about it */ - TCPWorkerThread worker(pipefds[1], crossProtocolFDs[1]); + TCPWorkerThread worker(pipefds[1], crossProtocolQueriesFDs[1], crossProtocolResponsesFDs[1]); try { - std::thread t1(tcpClientThread, pipefds[0], crossProtocolFDs[0]); + std::thread t1(tcpClientThread, pipefds[0], crossProtocolQueriesFDs[0], crossProtocolResponsesFDs[0], crossProtocolResponsesFDs[1]); t1.detach(); } catch (const std::runtime_error& e) { /* the thread creation failed, don't leak */ errlog("Error creating a TCP thread: %s", e.what()); close(pipefds[0]); + close(crossProtocolQueriesFDs[0]); + close(crossProtocolResponsesFDs[0]); return; } @@ -242,7 +256,7 @@ static void handleResponseSent(std::shared_ptr& stat double udiff = ids.sentTime.udiff(); vinfolog("Got answer from %s, relayed to %s (%s, %d bytes), took %f usec", ds->remote.toStringWithPort(), ids.origRemote.toStringWithPort(), (state->d_handler.isTLS() ? "DoT" : "TCP"), currentResponse.d_buffer.size(), udiff); - ::handleResponseSent(ids, udiff, state->d_ci.remote, ds->remote, static_cast(currentResponse.d_buffer.size()), currentResponse.d_cleartextDH); + ::handleResponseSent(ids, udiff, state->d_ci.remote, ds->remote, static_cast(currentResponse.d_buffer.size()), currentResponse.d_cleartextDH, ds->getProtocol()); updateTCPLatency(ds, udiff); } @@ -295,17 +309,17 @@ void IncomingTCPConnectionState::resetForNewQuery() d_state = State::waitingForQuery; } -std::shared_ptr IncomingTCPConnectionState::getActiveDownstreamConnection(const std::shared_ptr& ds, const std::unique_ptr>& tlvs) +std::shared_ptr IncomingTCPConnectionState::getOwnedDownstreamConnection(const std::shared_ptr& ds, const std::unique_ptr>& tlvs) { - auto it = d_activeConnectionsToBackend.find(ds); - if (it == d_activeConnectionsToBackend.end()) { - DEBUGLOG("no active connection found for "<getName()); + auto it = d_ownedConnectionsToBackend.find(ds); + if (it == d_ownedConnectionsToBackend.end()) { + DEBUGLOG("no owned connection found for "<getName()); return nullptr; } for (auto& conn : it->second) { - if (conn->canAcceptNewQueries() && conn->matchesTLVs(tlvs)) { - DEBUGLOG("Got one active connection accepting more for "<getName()); + if (conn->canBeReused(true) && conn->matchesTLVs(tlvs)) { + DEBUGLOG("Got one owned connection accepting more for "<getName()); conn->setReused(); return conn; } @@ -315,9 +329,9 @@ std::shared_ptr IncomingTCPConnectionState::getActiveDow return nullptr; } -void IncomingTCPConnectionState::registerActiveDownstreamConnection(std::shared_ptr& conn) +void IncomingTCPConnectionState::registerOwnedDownstreamConnection(std::shared_ptr& conn) { - d_activeConnectionsToBackend[conn->getDS()].push_front(conn); + d_ownedConnectionsToBackend[conn->getDS()].push_front(conn); } /* called when the buffer has been set and the rules have been processed, and only from handleIO (sometimes indirectly via handleQuery) */ @@ -363,7 +377,12 @@ void IncomingTCPConnectionState::terminateClientConnection() d_queuedResponses.clear(); /* we have already released idle connections that could be reused, we don't care about the ones still waiting for responses */ - d_activeConnectionsToBackend.clear(); + for (auto& backend : d_ownedConnectionsToBackend) { + for (auto& conn : backend.second) { + conn->release(); + } + } + d_ownedConnectionsToBackend.clear(); /* meaning we will no longer be 'active' when the backend response or timeout comes in */ d_ioState.reset(); @@ -407,22 +426,25 @@ void IncomingTCPConnectionState::handleResponse(const struct timeval& now, TCPRe { std::shared_ptr state = shared_from_this(); - if (response.d_connection && response.d_connection->isIdle()) { - // if we have added a TCP Proxy Protocol payload to a connection, don't release it to the general pool yet, no one else will be able to use it anyway - if (response.d_connection->canBeReused()) { - auto& list = state->d_activeConnectionsToBackend.at(response.d_connection->getDS()); - - for (auto it = list.begin(); it != list.end(); ++it) { - if (*it == response.d_connection) { - try { - response.d_connection->release(); - DownstreamConnectionsManager::releaseDownstreamConnection(std::move(*it)); - } - catch (const std::exception& e) { - vinfolog("Error releasing connection: %s", e.what()); + if (response.d_connection && response.d_connection->getDS() && response.d_connection->getDS()->useProxyProtocol) { + // if we have added a TCP Proxy Protocol payload to a connection, don't release it to the general pool as no one else will be able to use it anyway + if (!response.d_connection->willBeReusable(true)) { + // if it can't be reused even by us, well + const auto connIt = state->d_ownedConnectionsToBackend.find(response.d_connection->getDS()); + if (connIt != state->d_ownedConnectionsToBackend.end()) { + auto& list = connIt->second; + + for (auto it = list.begin(); it != list.end(); ++it) { + if (*it == response.d_connection) { + try { + response.d_connection->release(); + } + catch (const std::exception& e) { + vinfolog("Error releasing connection: %s", e.what()); + } + list.erase(it); + break; } - list.erase(it); - break; } } } @@ -466,6 +488,94 @@ void IncomingTCPConnectionState::handleResponse(const struct timeval& now, TCPRe queueResponse(state, now, std::move(response)); } +struct TCPCrossProtocolResponse +{ + TCPCrossProtocolResponse(TCPResponse&& response, std::shared_ptr& state, const struct timeval& now): d_response(std::move(response)), d_state(state), d_now(now) + { + } + + TCPResponse d_response; + std::shared_ptr d_state; + struct timeval d_now; +}; + +class TCPCrossProtocolQuerySender : public TCPQuerySender +{ +public: + TCPCrossProtocolQuerySender(std::shared_ptr& state, int responseDescriptor): d_state(state), d_responseDesc(responseDescriptor) + { + } + + bool active() const override + { + return d_state->active(); + } + + const ClientState* getClientState() const override + { + return d_state->getClientState(); + } + + void handleResponse(const struct timeval& now, TCPResponse&& response) override + { + if (d_responseDesc == -1) { + throw std::runtime_error("Invalid pipe descriptor in TCP Cross Protocol Query Sender"); + } + + auto ptr = new TCPCrossProtocolResponse(std::move(response), d_state, now); + static_assert(sizeof(ptr) <= PIPE_BUF, "Writes up to PIPE_BUF are guaranteed not to be interleaved and to either fully succeed or fail"); + ssize_t sent = write(d_responseDesc, &ptr, sizeof(ptr)); + if (sent != sizeof(ptr)) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + ++g_stats.tcpCrossProtocolResponsePipeFull; + vinfolog("Unable to pass a cross-protocol response to the TCP worker thread because the pipe is full"); + } + else { + vinfolog("Unable to pass a cross-protocol response to the TCP worker thread because we couldn't write to the pipe: %s", stringerror()); + } + delete ptr; + } + } + + void handleXFRResponse(const struct timeval& now, TCPResponse&& response) override + { + handleResponse(now, std::move(response)); + } + + void notifyIOError(IDState&& query, const struct timeval& now) override + { + TCPResponse response(PacketBuffer(), std::move(query), nullptr); + handleResponse(now, std::move(response)); + } + +private: + std::shared_ptr d_state; + int d_responseDesc{-1}; +}; + +class TCPCrossProtocolQuery : public CrossProtocolQuery +{ +public: + TCPCrossProtocolQuery(PacketBuffer&& buffer, IDState&& ids, std::shared_ptr& ds, std::shared_ptr& sender): d_sender(sender) + { + query = InternalQuery(std::move(buffer), std::move(ids)); + downstream = ds; + proxyProtocolPayloadSize = 0; + } + + ~TCPCrossProtocolQuery() + { + } + + std::shared_ptr getTCPQuerySender() override + { + return d_sender; + } + +private: + std::shared_ptr d_sender; +}; + static void handleQuery(std::shared_ptr& state, const struct timeval& now) { if (state->d_querySize < sizeof(dnsheader)) { @@ -504,7 +614,7 @@ static void handleQuery(std::shared_ptr& state, cons struct timespec queryRealTime; gettime(&queryRealTime, true); - std::shared_ptr dnsCryptQuery{nullptr}; + std::unique_ptr dnsCryptQuery{nullptr}; auto dnsCryptResponse = checkDNSCryptQuery(*state->d_ci.cs, state->d_buffer, dnsCryptQuery, queryRealTime.tv_sec, true); if (dnsCryptResponse) { TCPResponse response; @@ -587,14 +697,32 @@ static void handleQuery(std::shared_ptr& state, cons IDState ids; setIDStateFromDNSQuestion(ids, dq, std::move(qname)); ids.origID = dh->id; + ids.cs = state->d_ci.cs; + + ++state->d_currentQueriesCount; + + std::string proxyProtocolPayload; + if (ds->isDoH()) { + vinfolog("Got query for %s|%s from %s (%s, %d bytes), relayed to %s", ids.qname.toLogString(), QType(ids.qtype).toString(), state->d_proxiedRemote.toStringWithPort(), (state->d_handler.isTLS() ? "DoT" : "TCP"), state->d_buffer.size(), ds->getName()); + + /* we need to do this _before_ creating the cross protocol query because + after that the buffer will have been moved */ + if (ds->useProxyProtocol) { + proxyProtocolPayload = getProxyProtocolPayload(dq); + } + + auto incoming = std::make_shared(state, state->d_threadData.crossProtocolResponsesPipe); + auto cpq = std::make_unique(std::move(state->d_buffer), std::move(ids), ds, incoming); + cpq->query.d_proxyProtocolPayload = std::move(proxyProtocolPayload); + + ds->passCrossProtocolQuery(std::move(cpq)); + return; + } prependSizeToTCPQuery(state->d_buffer, 0); auto downstreamConnection = state->getDownstreamConnection(ds, dq.proxyProtocolValues, now); - bool proxyProtocolPayloadAdded = false; - std::string proxyProtocolPayload; - if (ds->useProxyProtocol) { /* if we ever sent a TLV over a connection, we can never go back */ if (!state->d_proxyProtocolPayloadHasTLV) { @@ -602,11 +730,6 @@ static void handleQuery(std::shared_ptr& state, cons } proxyProtocolPayload = getProxyProtocolPayload(dq); - if (state->d_proxyProtocolPayloadHasTLV && downstreamConnection->isFresh()) { - /* we will not be able to reuse an existing connection anyway so let's add the payload right now */ - addProxyProtocol(state->d_buffer, proxyProtocolPayload); - proxyProtocolPayloadAdded = true; - } } if (dq.proxyProtocolValues) { @@ -614,14 +737,8 @@ static void handleQuery(std::shared_ptr& state, cons } TCPQuery query(std::move(state->d_buffer), std::move(ids)); - if (proxyProtocolPayloadAdded) { - query.d_proxyProtocolPayloadAdded = true; - } - else { - query.d_proxyProtocolPayload = std::move(proxyProtocolPayload); - } + query.d_proxyProtocolPayload = std::move(proxyProtocolPayload); - ++state->d_currentQueriesCount; vinfolog("Got query for %s|%s from %s (%s, %d bytes), relayed to %s", query.d_idstate.qname.toLogString(), QType(query.d_idstate.qtype).toString(), state->d_proxiedRemote.toStringWithPort(), (state->d_handler.isTLS() ? "DoT" : "TCP"), query.d_buffer.size(), ds->getName()); std::shared_ptr incoming = state; downstreamConnection->queueQuery(incoming, std::move(query)); @@ -782,7 +899,15 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptractive() && state->d_state != IncomingTCPConnectionState::State::idle) { - iostate = state->d_ioState->getState(); + if (state->d_ioState->isWaitingForRead()) { + iostate = IOState::NeedRead; + } + else if (state->d_ioState->isWaitingForWrite()) { + iostate = IOState::NeedWrite; + } + else { + iostate = IOState::Done; + } } } else { @@ -858,9 +983,9 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptrd_ci.cs->tcpDiedSendingResponse; } - if (state->d_ioState->getState() == IOState::NeedWrite || state->d_queriesCount == 0) { + if (state->d_ioState->isWaitingForWrite() || state->d_queriesCount == 0) { DEBUGLOG("Got an exception while handling TCP query: "<d_ioState->getState() == IOState::NeedRead ? "reading" : "writing"), state->d_ci.remote.toStringWithPort(), e.what()); + vinfolog("Got an exception while handling (%s) TCP query from %s: %s", (state->d_ioState->isWaitingForRead() ? "reading" : "writing"), state->d_ci.remote.toStringWithPort(), e.what()); } else { vinfolog("Closing TCP client connection with %s: %s", state->d_ci.remote.toStringWithPort(), e.what()); @@ -926,7 +1051,7 @@ void IncomingTCPConnectionState::handleTimeout(std::shared_ptrd_ci.remote.toStringWithPort()); DEBUGLOG("client timeout"); - DEBUGLOG("Processed "<d_queriesCount<<" queries, current count is "<d_currentQueriesCount<<", "<d_activeConnectionsToBackend.size()<<" active connections, "<d_queuedResponses.size()<<" response queued"); + DEBUGLOG("Processed "<d_queriesCount<<" queries, current count is "<d_currentQueriesCount<<", "<d_ownedConnectionsToBackend.size()<<" owned connections, "<d_queuedResponses.size()<<" response queued"); if (write || state->d_currentQueriesCount == 0) { ++state->d_ci.cs->tcpClientTimeouts; @@ -937,14 +1062,6 @@ void IncomingTCPConnectionState::handleTimeout(std::shared_ptrd_state = IncomingTCPConnectionState::State::idle; state->d_ioState->update(IOState::Done, handleIOCallback, state); - -#ifdef DEBUGLOG_ENABLED - for (const auto& active : state->d_activeConnectionsToBackend) { - for (const auto& conn: active.second) { - DEBUGLOG("Connection to "<getName()<<" is "<<(conn->isIdle() ? "idle" : "not idle")); - } - } -#endif } } @@ -1016,106 +1133,162 @@ static void handleCrossProtocolQuery(int pipefd, FDMultiplexer::funcparam_t& par delete tmp; tmp = nullptr; - auto downstream = DownstreamConnectionsManager::getConnectionToDownstream(threadData->mplexer, downstreamServer, now); + try { + auto downstream = t_downstreamTCPConnectionsManager.getConnectionToDownstream(threadData->mplexer, downstreamServer, now, std::string()); - prependSizeToTCPQuery(query.d_buffer, proxyProtocolPayloadSize); - downstream->queueQuery(tqs, std::move(query)); + prependSizeToTCPQuery(query.d_buffer, proxyProtocolPayloadSize); + downstream->queueQuery(tqs, std::move(query)); + } + catch (...) { + tqs->notifyIOError(std::move(query.d_idstate), now); + } } catch (...) { delete tmp; tmp = nullptr; - throw; } } -static void tcpClientThread(int pipefd, int crossProtocolPipeFD) +static void handleCrossProtocolResponse(int pipefd, FDMultiplexer::funcparam_t& param) { - /* we get launched with a pipe on which we receive file descriptors from clients that we own - from that point on */ + TCPCrossProtocolResponse* tmp{nullptr}; - setThreadName("dnsdist/tcpClie"); + ssize_t got = read(pipefd, &tmp, sizeof(tmp)); + if (got == 0) { + throw std::runtime_error("EOF while reading from the TCP cross-protocol response pipe (" + std::to_string(pipefd) + ") in " + std::string(isNonBlocking(pipefd) ? "non-blocking" : "blocking") + " mode"); + } + else if (got == -1) { + if (errno == EAGAIN || errno == EINTR) { + return; + } + throw std::runtime_error("Error while reading from the TCP cross-protocol response pipe (" + std::to_string(pipefd) + ") in " + std::string(isNonBlocking(pipefd) ? "non-blocking" : "blocking") + " mode:" + stringerror()); + } + else if (got != sizeof(tmp)) { + throw std::runtime_error("Partial read while reading from the TCP cross-protocol response pipe (" + std::to_string(pipefd) + ") in " + std::string(isNonBlocking(pipefd) ? "non-blocking" : "blocking") + " mode"); + } - TCPClientThreadData data; + auto response = std::move(*tmp); + delete tmp; + tmp = nullptr; - data.mplexer->addReadFD(pipefd, handleIncomingTCPQuery, &data); - data.mplexer->addReadFD(crossProtocolPipeFD, handleCrossProtocolQuery, &data); + try { + if (response.d_response.d_buffer.empty()) { + response.d_state->notifyIOError(std::move(response.d_response.d_idstate), response.d_now); + } + else if (response.d_response.d_idstate.qtype == QType::AXFR || response.d_response.d_idstate.qtype == QType::IXFR) { + response.d_state->handleXFRResponse(response.d_now, std::move(response.d_response)); + } + else { + response.d_state->handleResponse(response.d_now, std::move(response.d_response)); + } + } + catch (...) { + /* no point bubbling up from there */ + } +} - struct timeval now; - gettimeofday(&now, nullptr); - time_t lastTimeoutScan = now.tv_sec; - - for (;;) { - data.mplexer->run(&now); - - if (now.tv_sec > lastTimeoutScan) { - lastTimeoutScan = now.tv_sec; - auto expiredReadConns = data.mplexer->getTimeouts(now, false); - for (const auto& cbData : expiredReadConns) { - if (cbData.second.type() == typeid(std::shared_ptr)) { - auto state = boost::any_cast>(cbData.second); - if (cbData.first == state->d_handler.getDescriptor()) { - vinfolog("Timeout (read) from remote TCP client %s", state->d_ci.remote.toStringWithPort()); - state->handleTimeout(state, false); - } - } - else if (cbData.second.type() == typeid(std::shared_ptr)) { - auto conn = boost::any_cast>(cbData.second); - vinfolog("Timeout (read) from remote backend %s", conn->getBackendName()); - conn->handleTimeout(now, false); - } - } +static void tcpClientThread(int pipefd, int crossProtocolQueriesPipeFD, int crossProtocolResponsesListenPipeFD, int crossProtocolResponsesWritePipeFD) +{ + /* we get launched with a pipe on which we receive file descriptors from clients that we own + from that point on */ - auto expiredWriteConns = data.mplexer->getTimeouts(now, true); - for (const auto& cbData : expiredWriteConns) { - if (cbData.second.type() == typeid(std::shared_ptr)) { - auto state = boost::any_cast>(cbData.second); - if (cbData.first == state->d_handler.getDescriptor()) { - vinfolog("Timeout (write) from remote TCP client %s", state->d_ci.remote.toStringWithPort()); - state->handleTimeout(state, true); - } - } - else if (cbData.second.type() == typeid(std::shared_ptr)) { - auto conn = boost::any_cast>(cbData.second); - vinfolog("Timeout (write) from remote backend %s", conn->getBackendName()); - conn->handleTimeout(now, true); - } - } + setThreadName("dnsdist/tcpClie"); + + try { + TCPClientThreadData data; + /* this is the writing end! */ + data.crossProtocolResponsesPipe = crossProtocolResponsesWritePipeFD; + data.mplexer->addReadFD(pipefd, handleIncomingTCPQuery, &data); + data.mplexer->addReadFD(crossProtocolQueriesPipeFD, handleCrossProtocolQuery, &data); + data.mplexer->addReadFD(crossProtocolResponsesListenPipeFD, handleCrossProtocolResponse, &data); - if (g_tcpStatesDumpRequested > 0) { - /* just to keep things clean in the output, debug only */ - static std::mutex s_lock; - std::lock_guard lck(s_lock); - if (g_tcpStatesDumpRequested > 0) { - /* no race here, we took the lock so it can only be increased in the meantime */ - --g_tcpStatesDumpRequested; - errlog("Dumping the TCP states, as requested:"); - data.mplexer->runForAllWatchedFDs([](bool isRead, int fd, const FDMultiplexer::funcparam_t& param, struct timeval ttd) - { - struct timeval lnow; - gettimeofday(&lnow, nullptr); - if (ttd.tv_sec > 0) { - errlog("- Descriptor %d is in %s state, TTD in %d", fd, (isRead ? "read" : "write"), (ttd.tv_sec-lnow.tv_sec)); + struct timeval now; + gettimeofday(&now, nullptr); + time_t lastTimeoutScan = now.tv_sec; + + for (;;) { + data.mplexer->run(&now); + + try { + t_downstreamTCPConnectionsManager.cleanupClosedConnections(now); + + if (now.tv_sec > lastTimeoutScan) { + lastTimeoutScan = now.tv_sec; + auto expiredReadConns = data.mplexer->getTimeouts(now, false); + for (const auto& cbData : expiredReadConns) { + if (cbData.second.type() == typeid(std::shared_ptr)) { + auto state = boost::any_cast>(cbData.second); + if (cbData.first == state->d_handler.getDescriptor()) { + vinfolog("Timeout (read) from remote TCP client %s", state->d_ci.remote.toStringWithPort()); + state->handleTimeout(state, false); + } } - else { - errlog("- Descriptor %d is in %s state, no TTD set", fd, (isRead ? "read" : "write")); + else if (cbData.second.type() == typeid(std::shared_ptr)) { + auto conn = boost::any_cast>(cbData.second); + vinfolog("Timeout (read) from remote backend %s", conn->getBackendName()); + conn->handleTimeout(now, false); } + } - if (param.type() == typeid(std::shared_ptr)) { - auto state = boost::any_cast>(param); - errlog(" - %s", state->toString()); + auto expiredWriteConns = data.mplexer->getTimeouts(now, true); + for (const auto& cbData : expiredWriteConns) { + if (cbData.second.type() == typeid(std::shared_ptr)) { + auto state = boost::any_cast>(cbData.second); + if (cbData.first == state->d_handler.getDescriptor()) { + vinfolog("Timeout (write) from remote TCP client %s", state->d_ci.remote.toStringWithPort()); + state->handleTimeout(state, true); + } } - else if (param.type() == typeid(std::shared_ptr)) { - auto conn = boost::any_cast>(param); - errlog(" - %s", conn->toString()); + else if (cbData.second.type() == typeid(std::shared_ptr)) { + auto conn = boost::any_cast>(cbData.second); + vinfolog("Timeout (write) from remote backend %s", conn->getBackendName()); + conn->handleTimeout(now, true); } - else if (param.type() == typeid(TCPClientThreadData*)) { - errlog(" - Worker thread pipe"); + } + + if (g_tcpStatesDumpRequested > 0) { + /* just to keep things clean in the output, debug only */ + static std::mutex s_lock; + std::lock_guard lck(s_lock); + if (g_tcpStatesDumpRequested > 0) { + /* no race here, we took the lock so it can only be increased in the meantime */ + --g_tcpStatesDumpRequested; + errlog("Dumping the TCP states, as requested:"); + data.mplexer->runForAllWatchedFDs([](bool isRead, int fd, const FDMultiplexer::funcparam_t& param, struct timeval ttd) + { + struct timeval lnow; + gettimeofday(&lnow, nullptr); + if (ttd.tv_sec > 0) { + errlog("- Descriptor %d is in %s state, TTD in %d", fd, (isRead ? "read" : "write"), (ttd.tv_sec-lnow.tv_sec)); + } + else { + errlog("- Descriptor %d is in %s state, no TTD set", fd, (isRead ? "read" : "write")); + } + + if (param.type() == typeid(std::shared_ptr)) { + auto state = boost::any_cast>(param); + errlog(" - %s", state->toString()); + } + else if (param.type() == typeid(std::shared_ptr)) { + auto conn = boost::any_cast>(param); + errlog(" - %s", conn->toString()); + } + else if (param.type() == typeid(TCPClientThreadData*)) { + errlog(" - Worker thread pipe"); + } + }); } - }); + } } } + catch (const std::exception& e) { + errlog("Error in TCP worker thread: %s", e.what()); + } } } + catch (const std::exception& e) { + errlog("Fatal error in TCP worker thread: %s", e.what()); + } } /* spawn as many of these as required, they call Accept on a socket on which they will accept queries, and @@ -1147,8 +1320,8 @@ void tcpAcceptorThread(ClientState* cs) continue; } - if (concurrentConnections > cs->tcpMaxConcurrentConnections) { - cs->tcpMaxConcurrentConnections = concurrentConnections; + if (concurrentConnections > cs->tcpMaxConcurrentConnections.load()) { + cs->tcpMaxConcurrentConnections.store(concurrentConnections); } if (ci->fd < 0) { diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc index 051df2a5872e..5e551076cbcc 100644 --- a/pdns/dnsdist-web.cc +++ b/pdns/dnsdist-web.cc @@ -40,6 +40,7 @@ #include "gettime.hh" #include "htmlfiles.h" #include "threadname.hh" +#include "sodcrypto.hh" #include "sstuff.hh" struct WebserverConfig @@ -50,8 +51,8 @@ struct WebserverConfig } NetmaskGroup acl; - std::string password; - std::string apiKey; + std::unique_ptr password; + std::unique_ptr apiKey; boost::optional > customHeaders; bool statsRequireAuthentication{true}; }; @@ -151,6 +152,12 @@ const std::map MetricDefinitionStorage::metrics{ { "udp-noport-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp NoPorts") }, { "udp-recvbuf-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp RcvbufErrors") }, { "udp-sndbuf-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp SndbufErrors") }, + { "udp-csum-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp InCsumErrors") }, + { "udp6-in-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp6 Udp6InErrors") }, + { "udp6-recvbuf-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp6 Udp6RcvbufErrors") }, + { "udp6-sndbuf-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp6 Udp6SndbufErrors") }, + { "udp6-noport-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp6 Udp6NoPorts") }, + { "udp6-in-csum-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp6 Udp6InCsumErrors") }, { "tcp-listen-overflows", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/netstat ListenOverflows") }, { "proxy-protocol-invalid", MetricDefinition(PrometheusMetricType::counter, "Number of queries dropped because of an invalid Proxy Protocol header") }, }; @@ -196,21 +203,21 @@ static void apiSaveACL(const NetmaskGroup& nmg) apiWriteConfigFile("acl", content); } -static bool checkAPIKey(const YaHTTP::Request& req, const string& expectedApiKey) +static bool checkAPIKey(const YaHTTP::Request& req, const std::unique_ptr& apiKey) { - if (expectedApiKey.empty()) { + if (!apiKey) { return false; } const auto header = req.headers.find("x-api-key"); if (header != req.headers.end()) { - return (header->second == expectedApiKey); + return apiKey->matches(header->second); } return false; } -static bool checkWebPassword(const YaHTTP::Request& req, const string &expected_password) +static bool checkWebPassword(const YaHTTP::Request& req, const std::unique_ptr& password) { static const char basicStr[] = "basic "; @@ -226,7 +233,10 @@ static bool checkWebPassword(const YaHTTP::Request& req, const string &expected_ stringtok(cparts, plain, ":"); if (cparts.size() == 2) { - return cparts[1] == expected_password; + if (password) { + return password->matches(cparts.at(1)); + } + return true; } } @@ -1320,6 +1330,11 @@ void registerWebHandler(const std::string& endpoint, std::function apiKey) +void setWebserverAPIKey(std::unique_ptr&& apiKey) { auto config = g_webserverConfig.lock(); if (apiKey) { - config->apiKey = *apiKey; + config->apiKey = std::move(apiKey); } else { - config->apiKey.clear(); + config->apiKey.reset(); } } -void setWebserverPassword(const std::string& password) +void setWebserverPassword(std::unique_ptr&& password) { - g_webserverConfig.lock()->password = password; + g_webserverConfig.lock()->password = std::move(password); } void setWebserverACL(const std::string& acl) @@ -1501,7 +1516,7 @@ void dnsdistWebserverThread(int sock, const ComboAddress& local) setThreadName("dnsdist/webserv"); warnlog("Webserver launched on %s", local.toStringWithPort()); - if (g_webserverConfig.lock()->password.empty()) { + if (!g_webserverConfig.lock()->password) { warnlog("Webserver launched on %s without a password set!", local.toStringWithPort()); } diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc index 6f36fcb06a9e..4d43eadec286 100644 --- a/pdns/dnsdist.cc +++ b/pdns/dnsdist.cc @@ -51,6 +51,7 @@ #include "dnsdist-ecs.hh" #include "dnsdist-healthchecks.hh" #include "dnsdist-lua.hh" +#include "dnsdist-nghttp2.hh" #include "dnsdist-proxy-protocol.hh" #include "dnsdist-rings.hh" #include "dnsdist-secpoll.hh" @@ -131,7 +132,7 @@ Rings g_rings; QueryCount g_qcount; GlobalStateHolder g_dstates; -GlobalStateHolder> g_dynblockNMG; +GlobalStateHolder> g_dynblockNMG; GlobalStateHolder> g_dynblockSMT; DNSAction::Action g_dynBlockAction = DNSAction::Action::Drop; int g_udpTimeout{2}; @@ -389,7 +390,7 @@ static bool fixUpResponse(PacketBuffer& response, const DNSName& qname, uint16_t } #ifdef HAVE_DNSCRYPT -static bool encryptResponse(PacketBuffer& response, size_t maximumSize, bool tcp, std::shared_ptr dnsCryptQuery) +static bool encryptResponse(PacketBuffer& response, size_t maximumSize, bool tcp, std::unique_ptr& dnsCryptQuery) { if (dnsCryptQuery) { int res = dnsCryptQuery->encryptResponse(response, maximumSize, tcp); @@ -550,11 +551,11 @@ static void pickBackendSocketsReadyForReceiving(const std::shared_ptrmplexer.lock())->getAvailableFDs(ready, 1000); } -void handleResponseSent(const IDState& ids, double udiff, const ComboAddress& client, const ComboAddress& backend, unsigned int size, const dnsheader& cleartextDH) +void handleResponseSent(const IDState& ids, double udiff, const ComboAddress& client, const ComboAddress& backend, unsigned int size, const dnsheader& cleartextDH, dnsdist::Protocol protocol) { struct timespec ts; gettime(&ts); - g_rings.insertResponse(ts, client, ids.qname, ids.qtype, static_cast(udiff), size, cleartextDH, backend); + g_rings.insertResponse(ts, client, ids.qname, ids.qtype, static_cast(udiff), size, cleartextDH, backend, protocol); switch (cleartextDH.rcode) { case RCode::NXDomain: @@ -695,7 +696,7 @@ void responderThread(std::shared_ptr dss) double udiff = ids->sentTime.udiff(); vinfolog("Got answer from %s, relayed to %s, took %f usec", dss->remote.toStringWithPort(), ids->origRemote.toStringWithPort(), udiff); - handleResponseSent(*ids, udiff, *dr.remote, dss->remote, static_cast(got), cleartextDH); + handleResponseSent(*ids, udiff, *dr.remote, dss->remote, static_cast(got), cleartextDH, dss->getProtocol()); dss->latencyUsec = (127.0 * dss->latencyUsec / 128.0) + udiff/128.0; @@ -803,13 +804,15 @@ bool processRulesResult(const DNSAction::Action& action, DNSQuestion& dq, std::s return true; break; case DNSAction::Action::Truncate: - dq.getHeader()->tc = true; - dq.getHeader()->qr = true; - dq.getHeader()->ra = dq.getHeader()->rd; - dq.getHeader()->aa = false; - dq.getHeader()->ad = false; - ++g_stats.ruleTruncated; - return true; + if (!dq.overTCP()) { + dq.getHeader()->tc = true; + dq.getHeader()->qr = true; + dq.getHeader()->ra = dq.getHeader()->rd; + dq.getHeader()->aa = false; + dq.getHeader()->ad = false; + ++g_stats.ruleTruncated; + return true; + } break; case DNSAction::Action::HeaderModify: return true; @@ -839,7 +842,7 @@ bool processRulesResult(const DNSAction::Action& action, DNSQuestion& dq, std::s static bool applyRulesToQuery(LocalHolders& holders, DNSQuestion& dq, const struct timespec& now) { - g_rings.insertQuery(now, *dq.remote, *dq.qname, dq.qtype, dq.getData().size(), *dq.getHeader()); + g_rings.insertQuery(now, *dq.remote, *dq.qname, dq.qtype, dq.getData().size(), *dq.getHeader(), dq.getProtocol()); if (g_qcount.enabled) { string qname = (*dq.qname).toLogString(); @@ -858,13 +861,14 @@ static bool applyRulesToQuery(LocalHolders& holders, DNSQuestion& dq, const stru } } - if(auto got = holders.dynNMGBlock->lookup(*dq.remote)) { + /* the Dynamic Block mechanism supports address and port ranges, so we need to pass the full address and port */ + if (auto got = holders.dynNMGBlock->lookup(AddressAndPortRange(*dq.remote, dq.remote->isIPv4() ? 32 : 128, 16))) { auto updateBlockStats = [&got]() { ++g_stats.dynBlocked; got->second.blocks++; }; - if(now < got->second.until) { + if (now < got->second.until) { DNSAction::Action action = got->second.action; if (action == DNSAction::Action::None) { action = g_dynBlockAction; @@ -918,13 +922,13 @@ static bool applyRulesToQuery(LocalHolders& holders, DNSQuestion& dq, const stru } } - if(auto got = holders.dynSMTBlock->lookup(*dq.qname)) { + if (auto got = holders.dynSMTBlock->lookup(*dq.qname)) { auto updateBlockStats = [&got]() { ++g_stats.dynBlocked; got->blocks++; }; - if(now < got->until) { + if (now < got->until) { DNSAction::Action action = got->action; if (action == DNSAction::Action::None) { action = g_dynBlockAction; @@ -1060,14 +1064,14 @@ static bool isUDPQueryAcceptable(ClientState& cs, LocalHolders& holders, const s return true; } -bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::shared_ptr& dnsCryptQuery, time_t now, bool tcp) +bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::unique_ptr& dnsCryptQuery, time_t now, bool tcp) { if (cs.dnscryptCtx) { #ifdef HAVE_DNSCRYPT PacketBuffer response; - dnsCryptQuery = std::make_shared(cs.dnscryptCtx); + dnsCryptQuery = std::make_unique(cs.dnscryptCtx); - bool decrypted = handleDNSCryptQuery(query, dnsCryptQuery, tcp, now, response); + bool decrypted = handleDNSCryptQuery(query, *dnsCryptQuery, tcp, now, response); if (!decrypted) { if (response.size() > 0) { @@ -1123,7 +1127,7 @@ static bool prepareOutgoingResponse(LocalHolders& holders, ClientState& cs, DNSQ DNSResponse dr(dq.qname, dq.qtype, dq.qclass, dq.local, dq.remote, dq.getMutableData(), dq.protocol, dq.queryTime); dr.uniqueId = dq.uniqueId; - dr.qTag = dq.qTag; + dr.qTag = std::move(dq.qTag); dr.delayMsec = dq.delayMsec; if (!applyRulesToResponse(cacheHit ? holders.cacheHitRespRuleactions : holders.selfAnsweredRespRuleactions, dr)) { @@ -1309,9 +1313,9 @@ class UDPTCPCrossQuerySender : public TCPQuerySender return true; } - const ClientState& getClientState() override + const ClientState* getClientState() const override { - return d_cs; + return &d_cs; } void handleResponse(const struct timeval& now, TCPResponse&& response) override @@ -1333,7 +1337,7 @@ class UDPTCPCrossQuerySender : public TCPQuerySender dnsheader cleartextDH; memcpy(&cleartextDH, dr.getHeader(), sizeof(cleartextDH)); - if (!processResponse(response.d_buffer, localRespRuleActions, dr, false, false)) { + if (!processResponse(response.d_buffer, localRespRuleActions, dr, false, true)) { return; } @@ -1351,7 +1355,7 @@ class UDPTCPCrossQuerySender : public TCPQuerySender double udiff = ids.sentTime.udiff(); vinfolog("Got answer from %s, relayed to %s (UDP), took %f usec", d_ds->remote.toStringWithPort(), ids.origRemote.toStringWithPort(), udiff); - handleResponseSent(ids, udiff, *dr.remote, d_ds->remote, response.d_buffer.size(), cleartextDH); + handleResponseSent(ids, udiff, *dr.remote, d_ds->remote, response.d_buffer.size(), cleartextDH, d_ds->getProtocol()); d_ds->latencyUsec = (127.0 * d_ds->latencyUsec / 128.0) + udiff/128.0; @@ -1429,7 +1433,7 @@ static void processUDPQuery(ClientState& cs, LocalHolders& holders, const struct struct timespec queryRealTime; gettime(&queryRealTime, true); - std::shared_ptr dnsCryptQuery = nullptr; + std::unique_ptr dnsCryptQuery = nullptr; auto dnsCryptResponse = checkDNSCryptQuery(cs, query, dnsCryptQuery, queryRealTime.tv_sec, false); if (dnsCryptResponse) { sendUDPResponse(cs.udpFD, query, 0, dest, remote); @@ -1490,6 +1494,13 @@ static void processUDPQuery(ClientState& cs, LocalHolders& holders, const struct } if (ss->isTCPOnly()) { + std::string proxyProtocolPayload; + /* we need to do this _before_ creating the cross protocol query because + after that the buffer will have been moved */ + if (ss->useProxyProtocol) { + proxyProtocolPayload = getProxyProtocolPayload(dq); + } + IDState ids; ids.cs = &cs; ids.origFD = cs.udpFD; @@ -1502,13 +1513,10 @@ static void processUDPQuery(ClientState& cs, LocalHolders& holders, const struct ids.origDest = cs.local; } auto cpq = std::make_unique(std::move(query), std::move(ids), ss); + cpq->query.d_proxyProtocolPayload = std::move(proxyProtocolPayload); - if (g_tcpclientthreads && g_tcpclientthreads->passCrossProtocolQueryToThread(std::move(cpq))) { - return ; - } - else { - return; - } + ss->passCrossProtocolQuery(std::move(cpq)); + return; } unsigned int idOffset = (ss->idOffset++) % ss->idStates.size(); @@ -1587,9 +1595,9 @@ static void MultipleMessagesUDPClientThread(ClientState* cs, LocalHolders& holde }; const size_t vectSize = g_udpVectorSize; - auto recvData = std::unique_ptr(new MMReceiver[vectSize]); - auto msgVec = std::unique_ptr(new struct mmsghdr[vectSize]); - auto outMsgVec = std::unique_ptr(new struct mmsghdr[vectSize]); + auto recvData = std::make_unique(vectSize); + auto msgVec = std::make_unique(vectSize); + auto outMsgVec = std::make_unique(vectSize); /* the actual buffer is larger because: - we may have to add EDNS and/or ECS @@ -1840,7 +1848,7 @@ static void healthChecksThread() for(;;) { sleep(interval); - auto mplexer = std::shared_ptr(FDMultiplexer::getMultiplexerSilent()); + auto mplexer = std::unique_ptr(FDMultiplexer::getMultiplexerSilent()); auto states = g_dstates.getLocal(); // this points to the actual shared_ptrs! for(auto& dss : *states) { if (++dss->lastCheck < dss->checkInterval) { @@ -1892,12 +1900,12 @@ static void healthChecksThread() memset(&fake, 0, sizeof(fake)); fake.id = ids.origID; - g_rings.insertResponse(ts, ids.origRemote, ids.qname, ids.qtype, std::numeric_limits::max(), 0, fake, dss->remote); + g_rings.insertResponse(ts, ids.origRemote, ids.qname, ids.qtype, std::numeric_limits::max(), 0, fake, dss->remote, dss->getProtocol()); } } } - handleQueuedHealthChecks(mplexer); + handleQueuedHealthChecks(*mplexer); } } @@ -1999,6 +2007,8 @@ static void checkFileDescriptorsLimits(size_t udpBindsCount, size_t tcpBindsCoun } } +static bool g_warned_ipv6_recvpktinfo = false; + static void setUpLocalBind(std::unique_ptr& cs) { /* skip some warnings if there is an identical UDP context */ @@ -2032,9 +2042,13 @@ static void setUpLocalBind(std::unique_ptr& cs) if(!cs->tcp && IsAnyAddress(cs->local)) { int one=1; - setsockopt(fd, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one)); // linux supports this, so why not - might fail on other systems + (void)setsockopt(fd, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one)); // linux supports this, so why not - might fail on other systems #ifdef IPV6_RECVPKTINFO - setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one)); + if (cs->local.isIPv6() && setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one)) < 0 && + !g_warned_ipv6_recvpktinfo) { + warnlog("Warning: IPV6_RECVPKTINFO setsockopt failed: %s", stringerror()); + g_warned_ipv6_recvpktinfo = true; + } #endif } @@ -2074,7 +2088,7 @@ static void setUpLocalBind(std::unique_ptr& cs) } #ifdef HAVE_EBPF - if (g_defaultBPFFilter) { + if (g_defaultBPFFilter && !g_defaultBPFFilter->isExternal()) { cs->attachFilter(g_defaultBPFFilter); vinfolog("Attaching default BPF Filter to %s frontend %s", (!cs->tcp ? "UDP" : "TCP"), cs->local.toStringWithPort()); } @@ -2163,8 +2177,22 @@ static void usage() } #ifdef COVERAGE +static void cleanupLuaObjects() +{ + /* when our coverage mode is enabled, we need to make + that the Lua objects destroyed before the Lua contexts. */ + g_ruleactions.setState({}); + g_respruleactions.setState({}); + g_cachehitrespruleactions.setState({}); + g_selfansweredrespruleactions.setState({}); + g_dstates.setState({}); + g_policy.setState(ServerPolicy()); + clearWebHandlers(); +} + static void sighandler(int sig) { + cleanupLuaObjects(); exit(EXIT_SUCCESS); } #endif @@ -2321,6 +2349,9 @@ int main(int argc, char** argv) #endif #ifdef HAVE_LMDB cout<<"lmdb "; +#endif +#ifdef HAVE_NGHTTP2 + cout<<"outgoing-dns-over-https(nghttp2) "; #endif cout<<"protobuf "; #ifdef HAVE_RE2 @@ -2392,6 +2423,7 @@ int main(int argc, char** argv) // No exception was thrown infolog("Configuration '%s' OK!", g_cmdLine.config); #ifdef COVERAGE + cleanupLuaObjects(); exit(EXIT_SUCCESS); #else _exit(EXIT_SUCCESS); @@ -2540,15 +2572,20 @@ int main(int argc, char** argv) } if (!g_maxTCPClientThreads) { - g_maxTCPClientThreads = std::max(tcpBindsCount, static_cast(10)); + g_maxTCPClientThreads = static_cast(10); } else if (*g_maxTCPClientThreads == 0 && tcpBindsCount > 0) { warnlog("setMaxTCPClientThreads() has been set to 0 while we are accepting TCP connections, raising to 1"); g_maxTCPClientThreads = 1; } + /* we need to create the TCP worker threads before the + acceptor ones, otherwise we might crash when processing + the first TCP query */ g_tcpclientthreads = std::make_unique(*g_maxTCPClientThreads); + initDoHWorkers(); + for (auto& t : todo) { t(); } @@ -2556,9 +2593,10 @@ int main(int argc, char** argv) localPools = g_pools.getCopy(); /* create the default pool no matter what */ createPoolIfNotExists(localPools, ""); - if(g_cmdLine.remotes.size()) { - for(const auto& address : g_cmdLine.remotes) { - auto ret=std::make_shared(ComboAddress(address, 53)); + if (g_cmdLine.remotes.size()) { + for (const auto& address : g_cmdLine.remotes) { + auto ret = std::make_shared(ComboAddress(address, 53)); + ret->connectUDPSockets(1); addServerToPool(localPools, "", ret); if (ret->connected && !ret->threadStarted.test_and_set()) { ret->tid = thread(responderThread, ret); @@ -2568,14 +2606,14 @@ int main(int argc, char** argv) } g_pools.setState(localPools); - if(g_dstates.getLocal()->empty()) { + if (g_dstates.getLocal()->empty()) { errlog("No downstream servers defined: all packets will get dropped"); // you might define them later, but you need to know } checkFileDescriptorsLimits(udpBindsCount, tcpBindsCount); - auto mplexer = std::shared_ptr(FDMultiplexer::getMultiplexerSilent()); + auto mplexer = std::unique_ptr(FDMultiplexer::getMultiplexerSilent()); for(auto& dss : g_dstates.getCopy()) { // it is a copy, but the internal shared_ptrs are the real deal if (dss->availability == DownstreamState::Availability::Auto) { if (!queueHealthCheck(mplexer, dss, true)) { @@ -2584,14 +2622,7 @@ int main(int argc, char** argv) } } } - handleQueuedHealthChecks(mplexer, true); - - /* we need to create the TCP worker threads before the - acceptor ones, otherwise we might crash when processing - the first TCP query */ - while (!g_tcpclientthreads->hasReachedMaxThreads()) { - g_tcpclientthreads->addTCPClientThread(); - } + handleQueuedHealthChecks(*mplexer, true); for(auto& cs : g_frontends) { if (cs->dohFrontend != nullptr) { @@ -2647,6 +2678,7 @@ int main(int argc, char** argv) doConsole(); } #ifdef COVERAGE + cleanupLuaObjects(); exit(EXIT_SUCCESS); #else _exit(EXIT_SUCCESS); diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index 85fa96ea2e06..4576f843593a 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -121,6 +121,20 @@ struct DNSQuestion return !(protocol == dnsdist::Protocol::DoUDP || protocol == dnsdist::Protocol::DNSCryptUDP); } + void setTag(const std::string& key, std::string&& value) { + if (!qTag) { + qTag = std::make_unique(); + } + qTag->insert_or_assign(key, std::move(value)); + } + + void setTag(const std::string& key, const std::string& value) { + if (!qTag) { + qTag = std::make_unique(); + } + qTag->insert_or_assign(key, value); + } + protected: PacketBuffer& data; @@ -130,6 +144,8 @@ public: boost::optional subnet; std::string sni; /* Server Name Indication, if any (DoT or DoH) */ std::string poolname; + mutable std::shared_ptr > ednsOptions; + std::shared_ptr packetCache{nullptr}; const DNSName* qname{nullptr}; const ComboAddress* local{nullptr}; const ComboAddress* remote{nullptr}; @@ -138,11 +154,9 @@ public: is enabled */ const ComboAddress* hopLocal{nullptr}; /* the address dnsdist received the packet from, see above */ const ComboAddress* hopRemote{nullptr}; - std::shared_ptr qTag{nullptr}; + std::unique_ptr qTag{nullptr}; std::unique_ptr> proxyProtocolValues{nullptr}; - mutable std::shared_ptr > ednsOptions; - std::shared_ptr dnsCryptQuery{nullptr}; - std::shared_ptr packetCache{nullptr}; + std::unique_ptr dnsCryptQuery{nullptr}; const struct timespec* queryTime{nullptr}; struct DOHUnit* du{nullptr}; int delayMsec{0}; @@ -309,7 +323,7 @@ struct DynBlock bool bpf{false}; }; -extern GlobalStateHolder> g_dynblockNMG; +extern GlobalStateHolder> g_dynblockNMG; extern vector > g_confDelta; @@ -347,7 +361,11 @@ struct DNSDistStats stat_t securityStatus{0}; stat_t dohQueryPipeFull{0}; stat_t dohResponsePipeFull{0}; + stat_t outgoingDoHQueryPipeFull{0}; stat_t proxyProtocolInvalid{0}; + stat_t tcpQueryPipeFull{0}; + stat_t tcpCrossProtocolQueryPipeFull{0}; + stat_t tcpCrossProtocolResponsePipeFull{0}; double latencyAvg100{0}, latencyAvg1000{0}, latencyAvg10000{0}, latencyAvg1000000{0}; typedef std::function statfunction_t; @@ -383,10 +401,16 @@ struct DNSDistStats {"uptime", uptimeOfProcess}, {"real-memory-usage", getRealMemoryUsage}, {"special-memory-usage", getSpecialMemoryUsage}, - {"udp-in-errors", boost::bind(udpErrorStats, "udp-in-errors")}, - {"udp-noport-errors", boost::bind(udpErrorStats, "udp-noport-errors")}, - {"udp-recvbuf-errors", boost::bind(udpErrorStats, "udp-recvbuf-errors")}, - {"udp-sndbuf-errors", boost::bind(udpErrorStats, "udp-sndbuf-errors")}, + {"udp-in-errors", std::bind(udpErrorStats, "udp-in-errors")}, + {"udp-noport-errors", std::bind(udpErrorStats, "udp-noport-errors")}, + {"udp-recvbuf-errors", std::bind(udpErrorStats, "udp-recvbuf-errors")}, + {"udp-sndbuf-errors", std::bind(udpErrorStats, "udp-sndbuf-errors")}, + {"udp-in-csum-errors", std::bind(udpErrorStats, "udp-in-csum-errors")}, + {"udp6-in-errors", std::bind(udp6ErrorStats, "udp6-in-errors")}, + {"udp6-recvbuf-errors", std::bind(udp6ErrorStats, "udp6-recvbuf-errors")}, + {"udp6-sndbuf-errors", std::bind(udp6ErrorStats, "udp6-sndbuf-errors")}, + {"udp6-noport-errors", std::bind(udp6ErrorStats, "udp6-noport-errors")}, + {"udp6-in-csum-errors", std::bind(udp6ErrorStats, "udp6-in-csum-errors")}, {"tcp-listen-overflows", std::bind(tcpErrorStats, "ListenOverflows")}, {"noncompliant-queries", &nonCompliantQueries}, {"noncompliant-responses", &nonCompliantResponses}, @@ -405,6 +429,10 @@ struct DNSDistStats {"security-status", &securityStatus}, {"doh-query-pipe-full", &dohQueryPipeFull}, {"doh-response-pipe-full", &dohResponsePipeFull}, + {"outgoing-doh-query-pipe-full", &outgoingDoHQueryPipeFull}, + {"tcp-query-pipe-full", &tcpQueryPipeFull}, + {"tcp-cross-protocol-query-pipe-full", &tcpCrossProtocolQueryPipeFull}, + {"tcp-cross-protocol-response-pipe-full", &tcpCrossProtocolResponsePipeFull}, // Latency histogram {"latency-sum", &latencySum}, {"latency-count", getLatencyCount}, @@ -543,16 +571,10 @@ extern QueryCount g_qcount; struct ClientState { - ClientState(const ComboAddress& local_, bool isTCP_, bool doReusePort, int fastOpenQueue, const std::string& itfName, const std::set& cpus_): cpus(cpus_), local(local_), interface(itfName), fastOpenQueueSize(fastOpenQueue), tcp(isTCP_), reuseport(doReusePort) + ClientState(const ComboAddress& local_, bool isTCP_, bool doReusePort, int fastOpenQueue, const std::string& itfName, const std::set& cpus_): cpus(cpus_), interface(itfName), local(local_), fastOpenQueueSize(fastOpenQueue), tcp(isTCP_), reuseport(doReusePort) { } - std::set cpus; - ComboAddress local; - std::shared_ptr dnscryptCtx{nullptr}; - std::shared_ptr tlsFrontend{nullptr}; - std::shared_ptr dohFrontend{nullptr}; - std::string interface; stat_t queries{0}; mutable stat_t responses{0}; mutable stat_t tcpDiedReadingQuery{0}; @@ -576,6 +598,13 @@ struct ClientState pdns::stat_t_trait tcpAvgQueriesPerConnection{0.0}; /* in ms */ pdns::stat_t_trait tcpAvgConnectionDuration{0.0}; + std::set cpus; + std::string interface; + ComboAddress local; + std::shared_ptr dnscryptCtx{nullptr}; + std::shared_ptr tlsFrontend{nullptr}; + std::shared_ptr dohFrontend{nullptr}; + std::shared_ptr d_filter{nullptr}; size_t d_maxInFlightQueriesPerConn{1}; size_t d_tcpConcurrentConnectionsLimit{0}; int udpFD{-1}; @@ -602,9 +631,14 @@ struct ClientState return udpFD == -1; } + bool isDoH() const + { + return dohFrontend != nullptr; + } + bool hasTLS() const { - return tlsFrontend != nullptr || dohFrontend != nullptr; + return tlsFrontend != nullptr || (dohFrontend != nullptr && dohFrontend->isHTTPS()); } std::string getType() const @@ -612,7 +646,12 @@ struct ClientState std::string result = udpFD != -1 ? "UDP" : "TCP"; if (dohFrontend) { - result += " (DNS over HTTPS)"; + if (dohFrontend->isHTTPS()) { + result += " (DNS over HTTPS)"; + } + else { + result += " (DNS over HTTP)"; + } } else if (tlsFrontend) { result += " (DNS over TLS)"; @@ -624,8 +663,6 @@ struct ClientState return result; } - shared_ptr d_filter; - void detachFilter() { if (d_filter) { @@ -649,33 +686,17 @@ struct ClientState } }; +struct CrossProtocolQuery; + struct DownstreamState { typedef std::function(const DNSName&, uint16_t, uint16_t, dnsheader*)> checkfunc_t; - DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf, const std::string& sourceItfName, size_t numberOfSockets, bool connect); - DownstreamState(const ComboAddress& remote_): DownstreamState(remote_, ComboAddress(), 0, std::string(), 1, true) {} + DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf, const std::string& sourceItfName); + DownstreamState(const ComboAddress& remote_): DownstreamState(remote_, ComboAddress(), 0, std::string()) {} ~DownstreamState(); + void connectUDPSockets(size_t numberOfSockets); - boost::uuids::uuid id; - SharedLockGuarded> hashes; - std::vector sockets; - const std::string sourceItfName; - std::string d_tlsSubjectName; - std::mutex connectLock; - LockGuarded> mplexer{nullptr}; - std::shared_ptr d_tlsCtx{nullptr}; - std::thread tid; - const ComboAddress remote; - QPSLimiter qps; - vector idStates; - const ComboAddress sourceAddr; - checkfunc_t checkFunction; - DNSName checkName{"a.root-servers.net."}; - QType checkType{QType::A}; - uint16_t checkClass{QClass::IN}; - std::atomic idOffset{0}; - std::atomic hashesComputed{false}; stat_t sendErrors{0}; stat_t outstanding{0}; stat_t reuseds{0}; @@ -702,11 +723,34 @@ struct DownstreamState pdns::stat_t_trait tcpAvgQueriesPerConnection{0.0}; /* in ms */ pdns::stat_t_trait tcpAvgConnectionDuration{0.0}; + pdns::stat_t_trait queryLoad{0.0}; + pdns::stat_t_trait dropRate{0.0}; + boost::uuids::uuid id; + const ComboAddress remote; + const ComboAddress sourceAddr; + SharedLockGuarded> hashes; + LockGuarded> mplexer{nullptr}; + const std::string sourceItfName; + std::string d_tlsSubjectName; + std::string d_dohPath; +private: + std::string name; + std::string nameWithAddr; +public: + std::shared_ptr d_tlsCtx{nullptr}; + std::vector sockets; + vector idStates; + set pools; + std::mutex connectLock; + std::thread tid; + checkfunc_t checkFunction; + DNSName checkName{"a.root-servers.net."}; + StopWatch sw; + QPSLimiter qps; + std::atomic idOffset{0}; size_t socketsOffset{0}; size_t d_maxInFlightQueriesPerConn{1}; size_t d_tcpConcurrentConnectionsLimit{0}; - pdns::stat_t_trait queryLoad{0.0}; - pdns::stat_t_trait dropRate{0.0}; double latencyUsec{0.0}; double latencyUsecTCP{0.0}; int order{1}; @@ -717,16 +761,20 @@ struct DownstreamState unsigned int checkInterval{1}; unsigned int lastCheck{0}; const unsigned int sourceItf{0}; - uint16_t retries{5}; + QType checkType{QType::A}; + uint16_t checkClass{QClass::IN}; + uint16_t d_retries{5}; uint16_t xpfRRCode{0}; uint16_t checkTimeout{1000}; /* in milliseconds */ uint8_t currentCheckFailures{0}; uint8_t consecutiveSuccessfulChecks{0}; uint8_t maxCheckFailures{1}; uint8_t minRiseSuccesses{1}; - StopWatch sw; - set pools; enum class Availability : uint8_t { Up, Down, Auto} availability{Availability::Auto}; +private: + bool d_stopped{false}; +public: + std::atomic hashesComputed{false}; bool mustResolve{false}; bool upStatus{false}; bool useECS{false}; @@ -740,6 +788,7 @@ struct DownstreamState bool reconnectOnUp{false}; bool d_tcpCheck{false}; bool d_tcpOnly{false}; + bool d_addXForwardedHeaders{false}; // for DoH backends bool isUp() const { @@ -823,10 +872,25 @@ struct DownstreamState return d_tcpOnly || d_tlsCtx != nullptr; } -private: - std::string name; - std::string nameWithAddr; - bool d_stopped{false}; + bool isDoH() const + { + return !d_dohPath.empty(); + } + + bool passCrossProtocolQuery(std::unique_ptr&& cpq); + dnsdist::Protocol getProtocol() const + { + if (isDoH()) { + return dnsdist::Protocol::DoH; + } + if (d_tlsCtx != nullptr) { + return dnsdist::Protocol::DoT; + } + if (isTCPOnly()) { + return dnsdist::Protocol::DoTCP; + } + return dnsdist::Protocol::DoUDP; + } }; using servers_t =vector>; @@ -870,6 +934,7 @@ struct ServerPool std::shared_ptr packetCache{nullptr}; std::shared_ptr policy{nullptr}; + size_t poolLoad(); size_t countServers(bool upOnly); const std::shared_ptr getServers(); void addServer(shared_ptr& server); @@ -967,7 +1032,7 @@ struct LocalHolders LocalStateHolder > cacheHitRespRuleactions; LocalStateHolder > selfAnsweredRespRuleactions; LocalStateHolder servers; - LocalStateHolder > dynNMGBlock; + LocalStateHolder > dynNMGBlock; LocalStateHolder > dynSMTBlock; LocalStateHolder pools; }; @@ -975,7 +1040,6 @@ struct LocalHolders vector> setupLua(bool client, const std::string& config); void tcpAcceptorThread(ClientState* p); -void setMaxCachedTCPConnectionsPerDownstream(size_t max); #ifdef HAVE_DNS_OVER_HTTPS void dohThread(ClientState* cs); @@ -993,8 +1057,8 @@ bool processRulesResult(const DNSAction::Action& action, DNSQuestion& dq, std::s bool checkQueryHeaders(const struct dnsheader* dh); extern std::vector> g_dnsCryptLocals; -int handleDNSCryptQuery(PacketBuffer& packet, std::shared_ptr& query, bool tcp, time_t now, PacketBuffer& response); -bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::shared_ptr& dnsCryptQuery, time_t now, bool tcp); +int handleDNSCryptQuery(PacketBuffer& packet, DNSCryptQuery& query, bool tcp, time_t now, PacketBuffer& response); +bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::unique_ptr& dnsCryptQuery, time_t now, bool tcp); uint16_t getRandomDNSID(); @@ -1017,6 +1081,6 @@ void setIDStateFromDNSQuestion(IDState& ids, DNSQuestion& dq, DNSName&& qname); int pickBackendSocketForSending(std::shared_ptr& state); ssize_t udpClientSendRequestToBackend(const std::shared_ptr& ss, const int sd, const PacketBuffer& request, bool healthCheck = false); -void handleResponseSent(const IDState& ids, double udiff, const ComboAddress& client, const ComboAddress& backend, unsigned int size, const dnsheader& cleartextDH); +void handleResponseSent(const IDState& ids, double udiff, const ComboAddress& client, const ComboAddress& backend, unsigned int size, const dnsheader& cleartextDH, dnsdist::Protocol protocol); void carbonDumpThread(); diff --git a/pdns/dnsdistdist/Makefile.am b/pdns/dnsdistdist/Makefile.am index 07ddd1267a6f..f1a614026b55 100644 --- a/pdns/dnsdistdist/Makefile.am +++ b/pdns/dnsdistdist/Makefile.am @@ -5,6 +5,7 @@ AM_CPPFLAGS += $(SYSTEMD_CFLAGS) \ $(FSTRM_CFLAGS) \ $(YAHTTP_CFLAGS) \ $(NET_SNMP_CFLAGS) \ + $(NGHTTP2_CFLAGS) \ $(LIBCAP_CFLAGS) \ -I$(top_srcdir)/ext/protozero/include \ -DSYSCONFDIR=\"${sysconfdir}\" \ @@ -129,6 +130,7 @@ dnsdist_SOURCES = \ capabilities.cc capabilities.hh \ circular_buffer.hh \ connection-management.hh \ + credentials.cc credentials.hh \ dns.cc dns.hh \ dnscrypt.cc dnscrypt.hh \ dnsdist-backend.cc \ @@ -158,6 +160,7 @@ dnsdist_SOURCES = \ dnsdist-lua-vars.cc \ dnsdist-lua-web.cc \ dnsdist-lua.cc dnsdist-lua.hh \ + dnsdist-nghttp2.cc dnsdist-nghttp2.hh \ dnsdist-prometheus.hh \ dnsdist-protobuf.cc dnsdist-protobuf.hh \ dnsdist-protocols.cc dnsdist-protocols.hh \ @@ -167,6 +170,7 @@ dnsdist_SOURCES = \ dnsdist-secpoll.cc dnsdist-secpoll.hh \ dnsdist-session-cache.cc dnsdist-session-cache.hh \ dnsdist-snmp.cc dnsdist-snmp.hh \ + dnsdist-svc.cc dnsdist-svc.hh \ dnsdist-systemd.cc dnsdist-systemd.hh \ dnsdist-tcp-downstream.cc dnsdist-tcp-downstream.hh \ dnsdist-tcp-upstream.hh \ @@ -224,6 +228,7 @@ testrunner_SOURCES = \ cachecleaner.hh \ circular_buffer.hh \ connection-management.hh \ + credentials.cc credentials.hh \ dns.cc dns.hh \ dnscrypt.cc dnscrypt.hh \ dnsdist-backend.cc \ @@ -240,11 +245,13 @@ testrunner_SOURCES = \ dnsdist-lua-ffi-interface.h dnsdist-lua-ffi-interface.inc \ dnsdist-lua-ffi.cc dnsdist-lua-ffi.hh \ dnsdist-lua-vars.cc \ + dnsdist-nghttp2.cc dnsdist-nghttp2.hh \ dnsdist-protocols.cc dnsdist-protocols.hh \ dnsdist-proxy-protocol.cc dnsdist-proxy-protocol.hh \ dnsdist-rings.cc dnsdist-rings.hh \ dnsdist-rules.cc dnsdist-rules.hh \ dnsdist-session-cache.cc dnsdist-session-cache.hh \ + dnsdist-svc.cc dnsdist-svc.hh \ dnsdist-tcp-downstream.cc \ dnsdist-tcp.cc dnsdist-tcp.hh \ dnsdist-xpf.cc dnsdist-xpf.hh \ @@ -275,15 +282,19 @@ testrunner_SOURCES = \ svc-records.cc svc-records.hh \ test-base64_cc.cc \ test-connectionmanagement_hh.cc \ + test-credentials_cc.cc \ test-delaypipe_hh.cc \ test-dnscrypt_cc.cc \ + test-dnsdist-connections-cache.cc \ test-dnsdist_cc.cc \ test-dnsdistdynblocks_hh.cc \ test-dnsdistkvs_cc.cc \ test-dnsdistlbpolicies_cc.cc \ + test-dnsdistnghttp2_cc.cc \ test-dnsdistpacketcache_cc.cc \ test-dnsdistrings_cc.cc \ test-dnsdistrules_cc.cc \ + test-dnsdistsvc_cc.cc \ test-dnsdisttcp_cc.cc \ test-dnsparser_cc.cc \ test-iputils_hh.cc \ @@ -343,6 +354,7 @@ endif if HAVE_LIBCRYPTO dnsdist_LDADD += $(LIBCRYPTO_LDFLAGS) $(LIBCRYPTO_LIBS) +testrunner_LDADD += $(LIBCRYPTO_LDFLAGS) $(LIBCRYPTO_LIBS) dnsdist_SOURCES += ipcipher.cc ipcipher.hh endif @@ -367,6 +379,11 @@ endif endif +if HAVE_NGHTTP2 +dnsdist_LDADD += $(NGHTTP2_LDFLAGS) $(NGHTTP2_LIBS) +testrunner_LDADD += $(NGHTTP2_LDFLAGS) $(NGHTTP2_LIBS) +endif + if !HAVE_LUA_HPP BUILT_SOURCES += lua.hpp nodist_dnsdist_SOURCES = lua.hpp @@ -507,6 +524,7 @@ endif dnsdist@.service: dnsdist.service $(AM_V_GEN)sed -e 's!/dnsdist !&--config $(sysconfdir)/dnsdist-%i.conf !' \ -e 's!RuntimeDirectory=.*!&-%i!' \ + -e 's!SyslogIdentifier=.*!&-%i!' \ < $< >$@ systemdsystemunitdir = $(SYSTEMD_DIR) diff --git a/pdns/dnsdistdist/configure.ac b/pdns/dnsdistdist/configure.ac index 02ac66043dd6..774ace1c45ec 100644 --- a/pdns/dnsdistdist/configure.ac +++ b/pdns/dnsdistdist/configure.ac @@ -96,6 +96,8 @@ AS_IF([test "x$enable_dns_over_https" != "xno"], [ ]) ]) +PDNS_WITH_NGHTTP2 + PDNS_CHECK_CDB PDNS_CHECK_LMDB @@ -224,6 +226,10 @@ AS_IF([test "x$enable_dns_over_tls" != "xno" -o "x$enable_dns_over_https" != "xn [AC_MSG_NOTICE([OpenSSL: no])] )] ) +AS_IF([test "x$NGHTTP2_LIBS" != "x"], + [AC_MSG_NOTICE([nghttp2: yes])], + [AC_MSG_NOTICE([nghttp2: no])] +) AS_IF([test "x$CDB_LIBS" != "x"], [AC_MSG_NOTICE([cdb: yes])], [AC_MSG_NOTICE([cdb: no])] diff --git a/pdns/dnsdistdist/credentials.cc b/pdns/dnsdistdist/credentials.cc new file mode 120000 index 000000000000..385b2128dfdc --- /dev/null +++ b/pdns/dnsdistdist/credentials.cc @@ -0,0 +1 @@ +../credentials.cc \ No newline at end of file diff --git a/pdns/dnsdistdist/credentials.hh b/pdns/dnsdistdist/credentials.hh new file mode 120000 index 000000000000..2c60bfff8f49 --- /dev/null +++ b/pdns/dnsdistdist/credentials.hh @@ -0,0 +1 @@ +../credentials.hh \ No newline at end of file diff --git a/pdns/dnsdistdist/dnsdist-backend.cc b/pdns/dnsdistdist/dnsdist-backend.cc index d0ffe2eecb26..2af756f8cccd 100644 --- a/pdns/dnsdistdist/dnsdist-backend.cc +++ b/pdns/dnsdistdist/dnsdist-backend.cc @@ -21,8 +21,21 @@ */ #include "dnsdist.hh" +#include "dnsdist-nghttp2.hh" +#include "dnsdist-tcp.hh" #include "dolog.hh" + +bool DownstreamState::passCrossProtocolQuery(std::unique_ptr&& cpq) +{ + if (d_dohPath.empty()) { + return g_tcpclientthreads && g_tcpclientthreads->passCrossProtocolQueryToThread(std::move(cpq)); + } + else { + return g_dohClientThreads && g_dohClientThreads->passCrossProtocolQueryToThread(std::move(cpq)); + } +} + bool DownstreamState::reconnect() { std::unique_lock tl(connectLock, std::try_to_lock); @@ -151,22 +164,28 @@ void DownstreamState::setWeight(int newWeight) } } -DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_, const std::string& sourceItfName_, size_t numberOfSockets, bool connect): sourceItfName(sourceItfName_), remote(remote_), idStates(connect ? g_maxOutstanding : 0), sourceAddr(sourceAddr_), sourceItf(sourceItf_), name(remote_.toStringWithPort()), nameWithAddr(remote_.toStringWithPort()) +DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_, const std::string& sourceItfName_): remote(remote_), sourceAddr(sourceAddr_), sourceItfName(sourceItfName_), name(remote_.toStringWithPort()), nameWithAddr(remote_.toStringWithPort()), sourceItf(sourceItf_) { id = getUniqueID(); threadStarted.clear(); - *(mplexer.lock()) = std::unique_ptr(FDMultiplexer::getMultiplexerSilent()); + sw.start(); +} +void DownstreamState::connectUDPSockets(size_t numberOfSockets) +{ + idStates.resize(g_maxOutstanding); sockets.resize(numberOfSockets); + + if (sockets.size() > 1) { + *(mplexer.lock()) = std::unique_ptr(FDMultiplexer::getMultiplexerSilent()); + } + for (auto& fd : sockets) { fd = -1; } - if (connect && !IsAnyAddress(remote)) { - reconnect(); - sw.start(); - } + reconnect(); } DownstreamState::~DownstreamState() @@ -205,6 +224,17 @@ size_t ServerPool::countServers(bool upOnly) return count; } +size_t ServerPool::poolLoad() +{ + size_t load = 0; + auto servers = d_servers.read_lock(); + for (const auto& server : **servers) { + size_t serverOutstanding = std::get<1>(server)->outstanding.load(); + load += serverOutstanding; + } + return load; +} + const std::shared_ptr ServerPool::getServers() { std::shared_ptr result; @@ -221,7 +251,7 @@ void ServerPool::addServer(shared_ptr& server) as other threads might hold a copy. We can however update the pointer as long as we hold the lock. */ unsigned int count = static_cast((*servers)->size()); auto newServers = std::make_shared(*(*servers)); - newServers->push_back(make_pair(++count, server)); + newServers->emplace_back(++count, server); /* we need to reorder based on the server 'order' */ std::stable_sort(newServers->begin(), newServers->end(), [](const std::pair >& a, const std::pair >& b) { return a.second->order < b.second->order; diff --git a/pdns/dnsdistdist/dnsdist-dynblocks.cc b/pdns/dnsdistdist/dnsdist-dynblocks.cc index fac0624264bc..3aa007830822 100644 --- a/pdns/dnsdistdist/dnsdist-dynblocks.cc +++ b/pdns/dnsdistdist/dnsdist-dynblocks.cc @@ -23,7 +23,7 @@ void DynBlockRulesGroup::apply(const struct timespec& now) return; } - boost::optional > blocks; + boost::optional > blocks; bool updated = false; for (const auto& entry : counts) { @@ -104,20 +104,27 @@ void DynBlockRulesGroup::apply(const struct timespec& now) if (!statNodeRoot.empty()) { StatNode::Stat node; - std::unordered_set namesToBlock; + std::unordered_map> namesToBlock; statNodeRoot.visit([this,&namesToBlock](const StatNode* node_, const StatNode::Stat& self, const StatNode::Stat& children) { bool block = false; + std::optional reason; if (d_smtVisitorFFI) { - dnsdist_ffi_stat_node_t tmp(*node_, self, children); + dnsdist_ffi_stat_node_t tmp(*node_, self, children, reason); block = d_smtVisitorFFI(&tmp); } else { - block = d_smtVisitor(*node_, self, children); + auto ret = d_smtVisitor(*node_, self, children); + block = std::get<0>(ret); + if (block) { + if (boost::optional tmp = std::get<1>(ret)) { + reason = std::move(*tmp); + } + } } if (block) { - namesToBlock.insert(DNSName(node_->fullname)); + namesToBlock.insert({DNSName(node_->fullname), std::move(reason)}); } }, node); @@ -125,8 +132,15 @@ void DynBlockRulesGroup::apply(const struct timespec& now) if (!namesToBlock.empty()) { updated = false; SuffixMatchTree smtBlocks = g_dynblockSMT.getCopy(); - for (const auto& name : namesToBlock) { - addOrRefreshBlockSMT(smtBlocks, now, name, d_suffixMatchRule, updated); + for (auto& [name, reason] : namesToBlock) { + if (reason) { + DynBlockRule rule(d_suffixMatchRule); + rule.d_blockReason = std::move(*reason); + addOrRefreshBlockSMT(smtBlocks, now, std::move(name), std::move(rule), updated); + } + else { + addOrRefreshBlockSMT(smtBlocks, now, std::move(name), d_suffixMatchRule, updated); + } } if (updated) { g_dynblockSMT.setState(std::move(smtBlocks)); @@ -160,9 +174,10 @@ bool DynBlockRulesGroup::checkIfResponseCodeMatches(const Rings::Response& respo return false; } -void DynBlockRulesGroup::addOrRefreshBlock(boost::optional >& blocks, const struct timespec& now, const ComboAddress& requestor, const DynBlockRule& rule, bool& updated, bool warning) +void DynBlockRulesGroup::addOrRefreshBlock(boost::optional >& blocks, const struct timespec& now, const AddressAndPortRange& requestor, const DynBlockRule& rule, bool& updated, bool warning) { - if (d_excludedSubnets.match(requestor)) { + /* network exclusions are address-based only (no port) */ + if (d_excludedSubnets.match(requestor.getNetwork())) { /* do not add a block for excluded subnets */ return; } @@ -173,7 +188,7 @@ void DynBlockRulesGroup::addOrRefreshBlock(boost::optional struct timespec until = now; until.tv_sec += rule.d_blockDuration; unsigned int count = 0; - const auto& got = blocks->lookup(Netmask(requestor)); + const auto& got = blocks->lookup(requestor); bool expired = false; bool wasWarning = false; bool bpf = false; @@ -209,10 +224,16 @@ void DynBlockRulesGroup::addOrRefreshBlock(boost::optional db.blocks = count; db.warning = warning; if (!got || expired || wasWarning) { - if (db.action == DNSAction::Action::Drop && g_defaultBPFFilter) { + if (g_defaultBPFFilter && + ((requestor.isIPv4() && requestor.getBits() == 32) || (requestor.isIPv6() && requestor.getBits() == 128)) && + (db.action == DNSAction::Action::Drop || db.action == DNSAction::Action::Truncate)) { try { - g_defaultBPFFilter->block(requestor); - bpf = true; + BPFFilter::MatchAction action = db.action == DNSAction::Action::Drop ? BPFFilter::MatchAction::Drop : BPFFilter::MatchAction::Truncate; + if (g_defaultBPFFilter->supportsMatchAction(action)) { + /* the current BPF filter implementation only supports full addresses (/32 or /128) and no port */ + g_defaultBPFFilter->block(requestor.getNetwork(), action); + bpf = true; + } } catch (const std::exception& e) { vinfolog("Unable to insert eBPF dynamic block for %s, falling back to regular dynamic block: %s", requestor.toString(), e.what()); @@ -226,7 +247,7 @@ void DynBlockRulesGroup::addOrRefreshBlock(boost::optional db.bpf = bpf; - blocks->insert(Netmask(requestor)).second = std::move(db); + blocks->insert(requestor).second = std::move(db); updated = true; } @@ -300,7 +321,7 @@ void DynBlockRulesGroup::processQueryRules(counts_t& counts, const struct timesp bool typeRuleMatches = checkIfQueryTypeMatches(c); if (qRateMatches || typeRuleMatches) { - auto& entry = counts[c.requestor]; + auto& entry = counts[AddressAndPortRange(c.requestor, c.requestor.isIPv4() ? d_v4Mask : d_v6Mask, d_portMask)]; if (qRateMatches) { ++entry.queries; } @@ -359,7 +380,7 @@ void DynBlockRulesGroup::processResponseRules(counts_t& counts, StatNode& root, continue; } - auto& entry = counts[c.requestor]; + auto& entry = counts[AddressAndPortRange(c.requestor, c.requestor.isIPv4() ? d_v4Mask : d_v6Mask, d_portMask)]; ++entry.responses; bool respRateMatches = d_respRateRule.matches(c.when); @@ -386,7 +407,7 @@ void DynBlockMaintenance::purgeExpired(const struct timespec& now) { { auto blocks = g_dynblockNMG.getLocal(); - std::vector toRemove; + std::vector toRemove; for (const auto& entry : *blocks) { if (!(now < entry.second.until)) { toRemove.push_back(entry.first); @@ -427,9 +448,9 @@ void DynBlockMaintenance::purgeExpired(const struct timespec& now) } } -std::map>> DynBlockMaintenance::getTopNetmasks(size_t topN) +std::map>> DynBlockMaintenance::getTopNetmasks(size_t topN) { - std::map>> results; + std::map>> results; if (topN == 0) { return results; } @@ -444,13 +465,13 @@ std::map>> DynBlockMaint } if (topsForReason.size() < topN || topsForReason.front().second < value) { - auto newEntry = std::make_pair(entry.first, value); + auto newEntry = std::pair(entry.first, value); if (topsForReason.size() >= topN) { topsForReason.pop_front(); } - topsForReason.insert(std::lower_bound(topsForReason.begin(), topsForReason.end(), newEntry, [](const std::pair& a, const std::pair& b) { + topsForReason.insert(std::lower_bound(topsForReason.begin(), topsForReason.end(), newEntry, [](const std::pair& a, const std::pair& b) { return a.second < b.second; }), newEntry); @@ -471,7 +492,7 @@ std::map>> DynBlockMaint blocks->visit([&results, topN](const SuffixMatchTree& node) { auto& topsForReason = results[node.d_value.reason]; if (topsForReason.size() < topN || topsForReason.front().second < node.d_value.blocks) { - auto newEntry = std::make_pair(node.d_value.domain, node.d_value.blocks.load()); + auto newEntry = std::pair(node.d_value.domain, node.d_value.blocks.load()); if (topsForReason.size() >= topN) { topsForReason.pop_front(); @@ -520,7 +541,7 @@ void DynBlockMaintenance::generateMetrics() } /* do NMG */ - std::map> nm; + std::map> nm; for (const auto& reason : s_metricsData.front().nmgData) { auto& reasonStat = nm[reason.first]; @@ -558,19 +579,19 @@ void DynBlockMaintenance::generateMetrics() } /* now we need to get the top N entries (for each "reason") based on our counters (sum of the last N entries) */ - std::map>> topNMGs; + std::map>> topNMGs; { for (const auto& reason : nm) { auto& topsForReason = topNMGs[reason.first]; for (const auto& entry : reason.second) { if (topsForReason.size() < s_topN || topsForReason.front().second < entry.second.sum) { /* Note that this is a gauge, so we need to divide by the number of elapsed seconds */ - auto newEntry = std::pair(entry.first, std::round(entry.second.sum / 60.0)); + auto newEntry = std::pair(entry.first, std::round(entry.second.sum / 60.0)); if (topsForReason.size() >= s_topN) { topsForReason.pop_front(); } - topsForReason.insert(std::lower_bound(topsForReason.begin(), topsForReason.end(), newEntry, [](const std::pair& a, const std::pair& b) { + topsForReason.insert(std::lower_bound(topsForReason.begin(), topsForReason.end(), newEntry, [](const std::pair& a, const std::pair& b) { return a.second < b.second; }), newEntry); @@ -707,7 +728,7 @@ void DynBlockMaintenance::run() } } -std::map>> DynBlockMaintenance::getHitsForTopNetmasks() +std::map>> DynBlockMaintenance::getHitsForTopNetmasks() { return s_tops.lock()->topNMGsByReason; } diff --git a/pdns/dnsdistdist/dnsdist-healthchecks.cc b/pdns/dnsdistdist/dnsdist-healthchecks.cc index acae3cd63ef5..ec9ab9902ffa 100644 --- a/pdns/dnsdistdist/dnsdist-healthchecks.cc +++ b/pdns/dnsdistdist/dnsdist-healthchecks.cc @@ -24,6 +24,8 @@ #include "tcpiohandler-mplexer.hh" #include "dnswriter.hh" #include "dolog.hh" +#include "dnsdist-tcp.hh" +#include "dnsdist-nghttp2.hh" bool g_verboseHealthChecks{false}; @@ -31,12 +33,12 @@ struct HealthCheckData { enum class TCPState : uint8_t { WritingQuery, ReadingResponseSize, ReadingResponse }; - HealthCheckData(std::shared_ptr& mplexer, const std::shared_ptr& ds, DNSName&& checkName, uint16_t checkType, uint16_t checkClass, uint16_t queryID): d_ds(ds), d_mplexer(mplexer), d_udpSocket(-1), d_checkName(std::move(checkName)), d_checkType(checkType), d_checkClass(checkClass), d_queryID(queryID) + HealthCheckData(FDMultiplexer& mplexer, const std::shared_ptr& ds, DNSName&& checkName, uint16_t checkType, uint16_t checkClass, uint16_t queryID): d_ds(ds), d_mplexer(mplexer), d_udpSocket(-1), d_checkName(std::move(checkName)), d_checkType(checkType), d_checkClass(checkClass), d_queryID(queryID) { } const std::shared_ptr d_ds; - std::shared_ptr d_mplexer; + FDMultiplexer& d_mplexer; std::unique_ptr d_tcpHandler{nullptr}; std::unique_ptr d_ioState{nullptr}; PacketBuffer d_buffer; @@ -83,7 +85,7 @@ void updateHealthCheckResult(const std::shared_ptr& dss, bool i dss->currentCheckFailures++; if (dss->currentCheckFailures < dss->maxCheckFailures) { /* we need more than one failure to be marked as down, - and we did not reach the threshold yet, let's stay down */ + and we did not reach the threshold yet, let's stay up */ newState = true; } } @@ -92,7 +94,7 @@ void updateHealthCheckResult(const std::shared_ptr& dss, bool i if (newState != dss->upStatus) { warnlog("Marking downstream %s as '%s'", dss->getNameWithAddr(), newState ? "up" : "down"); - if (newState && (!dss->connected || dss->reconnectOnUp)) { + if (newState && !dss->isTCPOnly() && (!dss->connected || dss->reconnectOnUp)) { newState = dss->reconnect(); if (dss->connected && !dss->threadStarted.test_and_set()) { @@ -178,10 +180,51 @@ static bool handleResponse(std::shared_ptr& data) return true; } +class HealthCheckQuerySender : public TCPQuerySender +{ +public: + HealthCheckQuerySender(std::shared_ptr& data): d_data(data) + { + } + + ~HealthCheckQuerySender() + { + } + + bool active() const override + { + return true; + } + + const ClientState* getClientState() const override + { + return nullptr; + } + + void handleResponse(const struct timeval& now, TCPResponse&& response) override + { + d_data->d_buffer = std::move(response.d_buffer); + updateHealthCheckResult(d_data->d_ds, d_data->d_initial, ::handleResponse(d_data)); + } + + void handleXFRResponse(const struct timeval& now, TCPResponse&& response) override + { + throw std::runtime_error("Unexpected XFR reponse to a health check query"); + } + + void notifyIOError(IDState&& query, const struct timeval& now) override + { + updateHealthCheckResult(d_data->d_ds, d_data->d_initial, false); + } + +private: + std::shared_ptr d_data; +}; + static void healthCheckUDPCallback(int fd, FDMultiplexer::funcparam_t& param) { auto data = boost::any_cast>(param); - data->d_mplexer->removeReadFD(fd); + data->d_mplexer.removeReadFD(fd); ComboAddress from; from.sin4.sin_family = data->d_ds->remote.sin4.sin_family; @@ -253,18 +296,20 @@ static void healthCheckTCPCallback(int fd, FDMultiplexer::funcparam_t& param) ioGuard.release(); } catch (const std::exception& e) { + updateHealthCheckResult(data->d_ds, data->d_initial, false); if (g_verboseHealthChecks) { infolog("Error checking the health of backend %s: %s", data->d_ds->getNameWithAddr(), e.what()); } } catch (...) { + updateHealthCheckResult(data->d_ds, data->d_initial, false); if (g_verboseHealthChecks) { infolog("Unknown exception while checking the health of backend %s", data->d_ds->getNameWithAddr()); } } } -bool queueHealthCheck(std::shared_ptr& mplexer, const std::shared_ptr& ds, bool initialCheck) +bool queueHealthCheck(std::unique_ptr& mplexer, const std::shared_ptr& ds, bool initialCheck) { try { @@ -298,11 +343,14 @@ bool queueHealthCheck(std::shared_ptr& mplexer, const std::shared /* we need to compute that _before_ adding the proxy protocol payload */ uint16_t packetSize = packet.size(); + std::string proxyProtocolPayload; size_t proxyProtocolPayloadSize = 0; if (ds->useProxyProtocol) { - auto payload = makeLocalProxyHeader(); - proxyProtocolPayloadSize = payload.size(); - packet.insert(packet.begin(), payload.begin(), payload.end()); + proxyProtocolPayload = makeLocalProxyHeader(); + proxyProtocolPayloadSize = proxyProtocolPayload.size(); + if (!ds->isDoH()) { + packet.insert(packet.begin(), proxyProtocolPayload.begin(), proxyProtocolPayload.end()); + } } Socket sock(ds->remote.sin4.sin_family, ds->doHealthcheckOverTCP() ? SOCK_STREAM : SOCK_DGRAM); @@ -327,7 +375,7 @@ bool queueHealthCheck(std::shared_ptr& mplexer, const std::shared sock.bind(ds->sourceAddr); } - auto data = std::make_shared(mplexer, ds, std::move(checkName), checkType, checkClass, queryID); + auto data = std::make_shared(*mplexer, ds, std::move(checkName), checkType, checkClass, queryID); data->d_initial = initialCheck; gettimeofday(&data->d_ttd, nullptr); @@ -352,6 +400,14 @@ bool queueHealthCheck(std::shared_ptr& mplexer, const std::shared mplexer->addReadFD(data->d_udpSocket.getHandle(), &healthCheckUDPCallback, data, &data->d_ttd); } + else if (ds->isDoH()) { + InternalQuery query(std::move(packet), IDState()); + query.d_proxyProtocolPayload = std::move(proxyProtocolPayload); + auto sender = std::shared_ptr(new HealthCheckQuerySender(data)); + if (!sendH2Query(ds, mplexer, sender, std::move(query), true)) { + updateHealthCheckResult(data->d_ds, data->d_initial, false); + } + } else { data->d_tcpHandler = std::make_unique(ds->d_tlsSubjectName, sock.releaseHandle(), timeval{ds->checkTimeout,0}, ds->d_tlsCtx, time(nullptr)); data->d_ioState = std::make_unique(*mplexer, data->d_tcpHandler->getDescriptor()); @@ -381,7 +437,7 @@ bool queueHealthCheck(std::shared_ptr& mplexer, const std::shared } return false; } - catch(...) + catch (...) { if (g_verboseHealthChecks) { infolog("Unknown exception while checking the health of backend %s", ds->getNameWithAddr()); @@ -390,26 +446,33 @@ bool queueHealthCheck(std::shared_ptr& mplexer, const std::shared } } -void handleQueuedHealthChecks(std::shared_ptr& mplexer, bool initial) +void handleQueuedHealthChecks(FDMultiplexer& mplexer, bool initial) { - while (mplexer->getWatchedFDCount(false) > 0 || mplexer->getWatchedFDCount(true) > 0) { + while (mplexer.getWatchedFDCount(false) > 0 || mplexer.getWatchedFDCount(true) > 0) { struct timeval now; - int ret = mplexer->run(&now, 100); + int ret = mplexer.run(&now, 100); if (ret == -1) { if (g_verboseHealthChecks) { infolog("Error while waiting for the health check response from backends: %d", ret); } break; } - auto timeouts = mplexer->getTimeouts(now); + + handleH2Timeouts(mplexer, now); + + auto timeouts = mplexer.getTimeouts(now); for (const auto& timeout : timeouts) { + if (timeout.second.type() != typeid(std::shared_ptr)) { + continue; + } + auto data = boost::any_cast>(timeout.second); try { if (data->d_ioState) { data->d_ioState.reset(); } else { - mplexer->removeReadFD(timeout.first); + mplexer.removeReadFD(timeout.first); } if (g_verboseHealthChecks) { infolog("Timeout while waiting for the health check response from backend %s", data->d_ds->getNameWithAddr()); @@ -419,18 +482,21 @@ void handleQueuedHealthChecks(std::shared_ptr& mplexer, bool init } catch (const std::exception& e) { if (g_verboseHealthChecks) { - infolog("Error while delaing with a timeout for the health check response from backend %s: %s", data->d_ds->getNameWithAddr(), e.what()); + infolog("Error while dealing with a timeout for the health check response from backend %s: %s", data->d_ds->getNameWithAddr(), e.what()); } } catch (...) { if (g_verboseHealthChecks) { - infolog("Error while delaing with a timeout for the health check response from backend %s", data->d_ds->getNameWithAddr()); + infolog("Error while dealing with a timeout for the health check response from backend %s", data->d_ds->getNameWithAddr()); } } } - timeouts = mplexer->getTimeouts(now, true); + timeouts = mplexer.getTimeouts(now, true); for (const auto& timeout : timeouts) { + if (timeout.second.type() != typeid(std::shared_ptr)) { + continue; + } auto data = boost::any_cast>(timeout.second); try { data->d_ioState.reset(); @@ -442,12 +508,12 @@ void handleQueuedHealthChecks(std::shared_ptr& mplexer, bool init } catch (const std::exception& e) { if (g_verboseHealthChecks) { - infolog("Error while delaing with a timeout for the health check response from backend %s: %s", data->d_ds->getNameWithAddr(), e.what()); + infolog("Error while dealing with a timeout for the health check response from backend %s: %s", data->d_ds->getNameWithAddr(), e.what()); } } catch (...) { if (g_verboseHealthChecks) { - infolog("Error while delaing with a timeout for the health check response from backend %s", data->d_ds->getNameWithAddr()); + infolog("Error while dealing with a timeout for the health check response from backend %s", data->d_ds->getNameWithAddr()); } } } diff --git a/pdns/dnsdistdist/dnsdist-healthchecks.hh b/pdns/dnsdistdist/dnsdist-healthchecks.hh index 08ac59948d9b..04af75b0dbdf 100644 --- a/pdns/dnsdistdist/dnsdist-healthchecks.hh +++ b/pdns/dnsdistdist/dnsdist-healthchecks.hh @@ -28,6 +28,6 @@ extern bool g_verboseHealthChecks; void updateHealthCheckResult(const std::shared_ptr& dss, bool initial, bool newState); -bool queueHealthCheck(std::shared_ptr& mplexer, const std::shared_ptr& ds, bool initial=false); -void handleQueuedHealthChecks(std::shared_ptr& mplexer, bool initial=false); +bool queueHealthCheck(std::unique_ptr& mplexer, const std::shared_ptr& ds, bool initial=false); +void handleQueuedHealthChecks(FDMultiplexer& mplexer, bool initial=false); diff --git a/pdns/dnsdistdist/dnsdist-idstate.cc b/pdns/dnsdistdist/dnsdist-idstate.cc index 0f3eeb247bfe..286808c76506 100644 --- a/pdns/dnsdistdist/dnsdist-idstate.cc +++ b/pdns/dnsdistdist/dnsdist-idstate.cc @@ -53,7 +53,7 @@ void setIDStateFromDNSQuestion(IDState& ids, DNSQuestion& dq, DNSName&& qname) ids.ednsAdded = dq.ednsAdded; ids.ecsAdded = dq.ecsAdded; ids.useZeroScope = dq.useZeroScope; - ids.qTag = dq.qTag; + ids.qTag = std::move(dq.qTag); ids.dnssecOK = dq.dnssecOK; ids.uniqueId = std::move(dq.uniqueId); diff --git a/pdns/dnsdistdist/dnsdist-kvs.cc b/pdns/dnsdistdist/dnsdist-kvs.cc index ead0e1096553..c2b6272ac4a0 100644 --- a/pdns/dnsdistdist/dnsdist-kvs.cc +++ b/pdns/dnsdistdist/dnsdist-kvs.cc @@ -190,7 +190,7 @@ CDBKVStore::~CDBKVStore() { bool CDBKVStore::reload(const struct stat& st) { - auto newCDB = std::unique_ptr(new CDB(d_fname)); + auto newCDB = std::make_unique(d_fname); { *(d_cdb.write_lock()) = std::move(newCDB); } diff --git a/pdns/dnsdistdist/dnsdist-lua-bindings-packetcache.cc b/pdns/dnsdistdist/dnsdist-lua-bindings-packetcache.cc index 7cadee528bf2..f77051e41759 100644 --- a/pdns/dnsdistdist/dnsdist-lua-bindings-packetcache.cc +++ b/pdns/dnsdistdist/dnsdist-lua-bindings-packetcache.cc @@ -28,10 +28,12 @@ #include "dnsdist.hh" #include "dnsdist-lua.hh" +#include + void setupLuaBindingsPacketCache(LuaContext& luaCtx, bool client) { /* PacketCache */ - luaCtx.writeFunction("newPacketCache", [client](size_t maxEntries, boost::optional>> vars) { + luaCtx.writeFunction("newPacketCache", [client](size_t maxEntries, boost::optional>>>> vars) { bool keepStaleData = false; size_t maxTTL = 86400; @@ -43,7 +45,7 @@ void setupLuaBindingsPacketCache(LuaContext& luaCtx, bool client) bool dontAge = false; bool deferrableInsertLock = true; bool ecsParsing = false; - bool cookieHashing = false; + std::unordered_set optionsToSkip{EDNSOptionCode::COOKIE}; if (vars) { @@ -88,7 +90,14 @@ void setupLuaBindingsPacketCache(LuaContext& luaCtx, bool client) } if (vars->count("cookieHashing")) { - cookieHashing = boost::get((*vars)["cookieHashing"]); + if (boost::get((*vars)["cookieHashing"])) { + optionsToSkip.erase(EDNSOptionCode::COOKIE); + } + } + if (vars->count("skipOptions")) { + for (auto option: boost::get>>(vars->at("skipOptions"))) { + optionsToSkip.insert(option.second); + } } } @@ -106,7 +115,7 @@ void setupLuaBindingsPacketCache(LuaContext& luaCtx, bool client) auto res = std::make_shared(maxEntries, maxTTL, minTTL, tempFailTTL, maxNegativeTTL, staleTTL, dontAge, numberOfShards, deferrableInsertLock, ecsParsing); res->setKeepStaleData(keepStaleData); - res->setCookieHashing(cookieHashing); + res->setSkippedOptions(optionsToSkip); return res; }); diff --git a/pdns/dnsdistdist/dnsdist-lua-ffi.cc b/pdns/dnsdistdist/dnsdist-lua-ffi.cc index 67be9c8afa3e..25e36c218f40 100644 --- a/pdns/dnsdistdist/dnsdist-lua-ffi.cc +++ b/pdns/dnsdistdist/dnsdist-lua-ffi.cc @@ -409,11 +409,7 @@ void dnsdist_ffi_dnsquestion_unset_temp_failure_ttl(dnsdist_ffi_dnsquestion_t* d void dnsdist_ffi_dnsquestion_set_tag(dnsdist_ffi_dnsquestion_t* dq, const char* label, const char* value) { - if (!dq->dq->qTag) { - dq->dq->qTag = std::make_shared(); - } - - dq->dq->qTag->insert({label, value}); + dq->dq->setTag(label, value); } size_t dnsdist_ffi_dnsquestion_get_trailing_data(dnsdist_ffi_dnsquestion_t* dq, const char** out) diff --git a/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc b/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc index e4ebea66c4c7..13158faa035a 100644 --- a/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc +++ b/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.cc @@ -99,3 +99,8 @@ uint64_t dnsdist_ffi_stat_node_get_children_bytes_count(const dnsdist_ffi_stat_n { return node->children.bytes; } + +void dnsdist_ffi_state_node_set_reason(dnsdist_ffi_stat_node_t* node, const char* reason, size_t reasonSize) +{ + node->reason = std::string(reason, reasonSize); +} diff --git a/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.hh b/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.hh index 6988d451b87e..05f087f649d4 100644 --- a/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.hh +++ b/pdns/dnsdistdist/dnsdist-lua-inspection-ffi.hh @@ -40,4 +40,6 @@ extern "C" { uint64_t dnsdist_ffi_stat_node_get_children_servfails_count(const dnsdist_ffi_stat_node_t* node) __attribute__ ((visibility ("default"))); uint64_t dnsdist_ffi_stat_node_get_children_drops_count(const dnsdist_ffi_stat_node_t* node) __attribute__ ((visibility ("default"))); uint64_t dnsdist_ffi_stat_node_get_children_bytes_count(const dnsdist_ffi_stat_node_t* node) __attribute__ ((visibility ("default"))); + + void dnsdist_ffi_state_node_set_reason(dnsdist_ffi_stat_node_t* node, const char* reason, size_t reasonSize) __attribute__ ((visibility ("default"))); } diff --git a/pdns/dnsdistdist/dnsdist-nghttp2.cc b/pdns/dnsdistdist/dnsdist-nghttp2.cc new file mode 100644 index 000000000000..5df7f337c804 --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-nghttp2.cc @@ -0,0 +1,1205 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "config.h" + +#ifdef HAVE_NGHTTP2 +#include +#endif /* HAVE_NGHTTP2 */ + +#include "dnsdist-nghttp2.hh" +#include "dnsdist-tcp.hh" +#include "dnsdist-tcp-downstream.hh" + +#include "dolog.hh" +#include "iputils.hh" +#include "libssl.hh" +#include "noinitvector.hh" +#include "tcpiohandler.hh" +#include "threadname.hh" +#include "sstuff.hh" + +std::atomic g_dohStatesDumpRequested{0}; +std::unique_ptr g_dohClientThreads{nullptr}; +std::optional g_outgoingDoHWorkerThreads{std::nullopt}; + +#ifdef HAVE_NGHTTP2 +class DoHConnectionToBackend : public ConnectionToBackend +{ +public: + DoHConnectionToBackend(const std::shared_ptr& ds, std::unique_ptr& mplexer, const struct timeval& now, std::string&& proxyProtocolPayload); + + void handleTimeout(const struct timeval& now, bool write) override; + void queueQuery(std::shared_ptr& sender, TCPQuery&& query) override; + + std::string toString() const override + { + ostringstream o; + o << "DoH connection to backend " << (d_ds ? d_ds->getName() : "empty") << " over FD " << (d_handler ? std::to_string(d_handler->getDescriptor()) : "no socket") << ", " << getConcurrentStreamsCount() << " streams"; + return o.str(); + } + + void setHealthCheck(bool h) + { + d_healthCheckQuery = h; + } + + void stopIO() override; + bool reachedMaxConcurrentQueries() const override; + bool reachedMaxStreamID() const override; + bool isIdle() const override; + void release() override + { + } + +private: + static ssize_t send_callback(nghttp2_session* session, const uint8_t* data, size_t length, int flags, void* user_data); + static int on_frame_recv_callback(nghttp2_session* session, const nghttp2_frame* frame, void* user_data); + static int on_data_chunk_recv_callback(nghttp2_session* session, uint8_t flags, int32_t stream_id, const uint8_t* data, size_t len, void* user_data); + static int on_stream_close_callback(nghttp2_session* session, int32_t stream_id, uint32_t error_code, void* user_data); + static int on_header_callback(nghttp2_session* session, const nghttp2_frame* frame, const uint8_t* name, size_t namelen, const uint8_t* value, size_t valuelen, uint8_t flags, void* user_data); + static int on_error_callback(nghttp2_session* session, int lib_error_code, const char* msg, size_t len, void* user_data); + static void handleReadableIOCallback(int fd, FDMultiplexer::funcparam_t& param); + static void handleWritableIOCallback(int fd, FDMultiplexer::funcparam_t& param); + + static void addStaticHeader(std::vector& headers, const std::string& nameKey, const std::string& valueKey); + static void addDynamicHeader(std::vector& headers, const std::string& nameKey, const std::string& value); + + class PendingRequest + { + public: + std::shared_ptr d_sender{nullptr}; + TCPQuery d_query; + PacketBuffer d_buffer; + size_t d_queryPos{0}; + uint16_t d_responseCode{0}; + bool d_finished{false}; + }; + void addToIOState(IOState state, FDMultiplexer::callbackfunc_t callback); + void updateIO(IOState newState, FDMultiplexer::callbackfunc_t callback, bool noTTD = false); + void watchForRemoteHostClosingConnection(); + void handleResponse(PendingRequest&& request); + void handleResponseError(PendingRequest&& request, const struct timeval& now); + void handleIOError(); + uint32_t getConcurrentStreamsCount() const; + + size_t getUsageCount() const + { + auto ref = shared_from_this(); + return ref.use_count(); + } + + static const std::unordered_map s_constants; + + std::unordered_map d_currentStreams; + std::string d_proxyProtocolPayload; + PacketBuffer d_out; + PacketBuffer d_in; + std::unique_ptr d_session{nullptr, nghttp2_session_del}; + size_t d_outPos{0}; + size_t d_inPos{0}; + bool d_healthCheckQuery{false}; + bool d_firstWrite{true}; +}; + +using DownstreamDoHConnectionsManager = DownstreamConnectionsManager; +thread_local DownstreamDoHConnectionsManager t_downstreamDoHConnectionsManager; + +uint32_t DoHConnectionToBackend::getConcurrentStreamsCount() const +{ + return d_currentStreams.size(); +} + +void DoHConnectionToBackend::handleResponse(PendingRequest&& request) +{ + struct timeval now; + gettimeofday(&now, nullptr); + try { + request.d_sender->handleResponse(now, TCPResponse(std::move(request.d_buffer), std::move(request.d_query.d_idstate), shared_from_this())); + } + catch (const std::exception& e) { + vinfolog("Got exception while handling response for cross-protocol DoH: %s", e.what()); + } +} + +void DoHConnectionToBackend::handleResponseError(PendingRequest&& request, const struct timeval& now) +{ + try { + request.d_sender->notifyIOError(std::move(request.d_query.d_idstate), now); + } + catch (const std::exception& e) { + vinfolog("Got exception while handling response for cross-protocol DoH: %s", e.what()); + } +} + +void DoHConnectionToBackend::handleIOError() +{ + d_connectionDied = true; + nghttp2_session_terminate_session(d_session.get(), NGHTTP2_PROTOCOL_ERROR); + + struct timeval now; + gettimeofday(&now, nullptr); + for (auto& request : d_currentStreams) { + handleResponseError(std::move(request.second), now); + } + + d_currentStreams.clear(); + stopIO(); +} + +void DoHConnectionToBackend::handleTimeout(const struct timeval& now, bool write) +{ + if (write) { + if (d_firstWrite) { + ++d_ds->tcpConnectTimeouts; + } + else { + ++d_ds->tcpWriteTimeouts; + } + } + else { + ++d_ds->tcpReadTimeouts; + } + + handleIOError(); +} + +bool DoHConnectionToBackend::reachedMaxStreamID() const +{ + const uint32_t maximumStreamID = (static_cast(1) << 31) - 1; + return d_highestStreamID == maximumStreamID; +} + +bool DoHConnectionToBackend::reachedMaxConcurrentQueries() const +{ + //cerr<<"Got "< DoHConnectionToBackend::s_constants = { + {"method-name", ":method"}, + {"method-value", "POST"}, + {"scheme-name", ":scheme"}, + {"scheme-value", "https"}, + {"accept-name", "accept"}, + {"accept-value", "application/dns-message"}, + {"content-type-name", "content-type"}, + {"content-type-value", "application/dns-message"}, + {"user-agent-name", "user-agent"}, + {"user-agent-value", "nghttp2-" NGHTTP2_VERSION "/dnsdist"}, + {"authority-name", ":authority"}, + {"path-name", ":path"}, + {"content-length-name", "content-length"}, + {"x-forwarded-for-name", "x-forwarded-for"}, + {"x-forwarded-port-name", "x-forwarded-port"}, + {"x-forwarded-proto-name", "x-forwarded-proto"}, + {"x-forwarded-proto-value-dns-over-udp", "dns-over-udp"}, + {"x-forwarded-proto-value-dns-over-tcp", "dns-over-tcp"}, + {"x-forwarded-proto-value-dns-over-tls", "dns-over-tls"}, + {"x-forwarded-proto-value-dns-over-http", "dns-over-http"}, + {"x-forwarded-proto-value-dns-over-https", "dns-over-https"}, +}; + +void DoHConnectionToBackend::addStaticHeader(std::vector& headers, const std::string& nameKey, const std::string& valueKey) +{ + const auto& name = s_constants.at(nameKey); + const auto& value = s_constants.at(valueKey); + + headers.push_back({const_cast(reinterpret_cast(name.c_str())), const_cast(reinterpret_cast(value.c_str())), name.size(), value.size(), NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE}); +} + +void DoHConnectionToBackend::addDynamicHeader(std::vector& headers, const std::string& nameKey, const std::string& value) +{ + const auto& name = s_constants.at(nameKey); + + headers.push_back({const_cast(reinterpret_cast(name.c_str())), const_cast(reinterpret_cast(value.c_str())), name.size(), value.size(), NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE}); +} + +void DoHConnectionToBackend::queueQuery(std::shared_ptr& sender, TCPQuery&& query) +{ + auto payloadSize = std::to_string(query.d_buffer.size()); + + bool addXForwarded = d_ds->d_addXForwardedHeaders; + + /* We use nghttp2_nv_flag.NGHTTP2_NV_FLAG_NO_COPY_NAME and nghttp2_nv_flag.NGHTTP2_NV_FLAG_NO_COPY_VALUE + to avoid a copy and lowercasing but we need to make sure that the data will outlive the request (nghttp2_on_frame_send_callback called), and that it is already lowercased. */ + std::vector headers; + // these need to live until after the request headers have been processed + std::string remote; + std::string remotePort; + headers.reserve(8 + (addXForwarded ? 3 : 0)); + + /* Pseudo-headers need to come first (rfc7540 8.1.2.1) */ + addStaticHeader(headers, "method-name", "method-value"); + addStaticHeader(headers, "scheme-name", "scheme-value"); + addDynamicHeader(headers, "authority-name", d_ds->d_tlsSubjectName); + addDynamicHeader(headers, "path-name", d_ds->d_dohPath); + addStaticHeader(headers, "accept-name", "accept-value"); + addStaticHeader(headers, "content-type-name", "content-type-value"); + addStaticHeader(headers, "user-agent-name", "user-agent-value"); + addDynamicHeader(headers, "content-length-name", payloadSize); + /* no need to add these headers for health-check queries */ + if (addXForwarded && query.d_idstate.origRemote.getPort() != 0) { + remote = query.d_idstate.origRemote.toString(); + remotePort = std::to_string(query.d_idstate.origRemote.getPort()); + addDynamicHeader(headers, "x-forwarded-for-name", remote); + addDynamicHeader(headers, "x-forwarded-port-name", remotePort); + if (query.d_idstate.cs != nullptr) { + if (query.d_idstate.cs->isUDP()) { + addStaticHeader(headers, "x-forwarded-proto-name", "x-forwarded-proto-value-dns-over-udp"); + } + else if (query.d_idstate.cs->isDoH()) { + if (query.d_idstate.cs->hasTLS()) { + addStaticHeader(headers, "x-forwarded-proto-name", "x-forwarded-proto-value-dns-over-https"); + } + else { + addStaticHeader(headers, "x-forwarded-proto-name", "x-forwarded-proto-value-dns-over-http"); + } + } + else if (query.d_idstate.cs->hasTLS()) { + addStaticHeader(headers, "x-forwarded-proto-name", "x-forwarded-proto-value-dns-over-tls"); + } + else { + addStaticHeader(headers, "x-forwarded-proto-name", "x-forwarded-proto-value-dns-over-tcp"); + } + } + } + + PendingRequest pending; + pending.d_query = std::move(query); + pending.d_sender = std::move(sender); + + uint32_t streamId = nghttp2_session_get_next_stream_id(d_session.get()); + auto insertPair = d_currentStreams.insert({streamId, std::move(pending)}); + if (!insertPair.second) { + /* there is a stream ID collision, something is very wrong! */ + d_connectionDied = true; + nghttp2_session_terminate_session(d_session.get(), NGHTTP2_NO_ERROR); + throw std::runtime_error("Stream ID collision"); + } + + /* if data_prd is not NULL, it provides data which will be sent in subsequent DATA frames. In this case, a method that allows request message bodies (https://tools.ietf.org/html/rfc7231#section-4) must be specified with :method key (e.g. POST). This function does not take ownership of the data_prd. The function copies the members of the data_prd. If data_prd is NULL, HEADERS have END_STREAM set. + */ + nghttp2_data_provider data_provider; + + /* we will not use this pointer */ + data_provider.source.ptr = this; + data_provider.read_callback = [](nghttp2_session* session, int32_t stream_id, uint8_t* buf, size_t length, uint32_t* data_flags, nghttp2_data_source* source, void* user_data) -> ssize_t { + auto conn = reinterpret_cast(user_data); + auto& request = conn->d_currentStreams.at(stream_id); + size_t toCopy = 0; + if (request.d_queryPos < request.d_query.d_buffer.size()) { + size_t remaining = request.d_query.d_buffer.size() - request.d_queryPos; + toCopy = length > remaining ? remaining : length; + memcpy(buf, &request.d_query.d_buffer.at(request.d_queryPos), toCopy); + request.d_queryPos += toCopy; + } + + if (request.d_queryPos >= request.d_query.d_buffer.size()) { + *data_flags |= NGHTTP2_DATA_FLAG_EOF; + } + return toCopy; + }; + + auto newStreamId = nghttp2_submit_request(d_session.get(), nullptr, headers.data(), headers.size(), &data_provider, this); + if (newStreamId < 0) { + d_connectionDied = true; + ++d_ds->tcpDiedSendingQuery; + d_currentStreams.erase(streamId); + throw std::runtime_error("Error submitting HTTP request:" + std::string(nghttp2_strerror(newStreamId))); + } + + auto rv = nghttp2_session_send(d_session.get()); + if (rv != 0) { + d_connectionDied = true; + ++d_ds->tcpDiedSendingQuery; + d_currentStreams.erase(streamId); + throw std::runtime_error("Error in nghttp2_session_send:" + std::to_string(rv)); + } + + d_highestStreamID = newStreamId; +} + +class DoHClientThreadData +{ +public: + DoHClientThreadData() : + mplexer(std::unique_ptr(FDMultiplexer::getMultiplexerSilent())) + { + } + + std::unique_ptr mplexer{nullptr}; +}; + +void DoHConnectionToBackend::handleReadableIOCallback(int fd, FDMultiplexer::funcparam_t& param) +{ + auto conn = boost::any_cast>(param); + if (fd != conn->getHandle()) { + throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__PRETTY_FUNCTION__) + ", expected " + std::to_string(conn->getHandle())); + } + + IOStateGuard ioGuard(conn->d_ioState); + do { + conn->d_inPos = 0; + conn->d_in.resize(conn->d_in.size() + 512); + // cerr<<"trying to read "<d_in.size()<d_handler->tryRead(conn->d_in, conn->d_inPos, conn->d_in.size(), true); + // cerr<<"got a "<<(int)newState<<" state and "<d_inPos<<" bytes"<d_in.resize(conn->d_inPos); + + if (conn->d_inPos > 0) { + /* we got something */ + auto readlen = nghttp2_session_mem_recv(conn->d_session.get(), conn->d_in.data(), conn->d_inPos); + // cerr<<"nghttp2_session_mem_recv returned "< 0 && static_cast(readlen) < conn->d_inPos) { + throw std::runtime_error("Fatal error while passing received data to nghttp2: " + std::string(nghttp2_strerror((int)readlen))); + } + + struct timeval now; + gettimeofday(&now, nullptr); + conn->d_lastDataReceivedTime = now; + + // cerr<<"after read send"<d_session.get()); + } + + if (newState == IOState::Done) { + if (conn->isIdle()) { + conn->stopIO(); + conn->watchForRemoteHostClosingConnection(); + ioGuard.release(); + break; + } + } + else { + if (newState == IOState::NeedWrite) { + // cerr<<"need write"<updateIO(IOState::NeedWrite, handleReadableIOCallback); + } + ioGuard.release(); + break; + } + } + catch (const std::exception& e) { + vinfolog("Exception while trying to read from HTTP backend connection: %s", e.what()); + ++conn->d_ds->tcpDiedReadingResponse; + conn->handleIOError(); + break; + } + } while (conn->getConcurrentStreamsCount() > 0); +} + +void DoHConnectionToBackend::handleWritableIOCallback(int fd, FDMultiplexer::funcparam_t& param) +{ + auto conn = boost::any_cast>(param); + if (fd != conn->getHandle()) { + throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__PRETTY_FUNCTION__) + ", expected " + std::to_string(conn->getHandle())); + } + IOStateGuard ioGuard(conn->d_ioState); + + // cerr<<"in "<<__PRETTY_FUNCTION__<<" trying to write "<d_out.size()-conn->d_outPos<d_handler->tryWrite(conn->d_out, conn->d_outPos, conn->d_out.size()); + // cerr<<"got a "<<(int)newState<<" state, "<d_out.size()-conn->d_outPos<<" bytes remaining"<updateIO(IOState::NeedRead, handleWritableIOCallback); + } + else if (newState == IOState::Done) { + // cerr<<"done, buffer size was "<d_out.size()<<", pos was "<d_outPos<d_firstWrite = false; + conn->d_out.clear(); + conn->d_outPos = 0; + conn->stopIO(); + if (!conn->isIdle()) { + conn->updateIO(IOState::NeedRead, handleReadableIOCallback); + } + else { + conn->watchForRemoteHostClosingConnection(); + } + } + ioGuard.release(); + } + catch (const std::exception& e) { + vinfolog("Exception while trying to write (ready) to HTTP backend connection: %s", e.what()); + ++conn->d_ds->tcpDiedSendingQuery; + conn->handleIOError(); + } +} + +void DoHConnectionToBackend::stopIO() +{ + d_ioState->reset(); + + auto shared = std::dynamic_pointer_cast(shared_from_this()); + if (!willBeReusable(false)) { + /* remove ourselves from the connection cache, this might mean that our + reference count drops to zero after that, so we need to be careful */ + t_downstreamDoHConnectionsManager.removeDownstreamConnection(shared); + } + else { + t_downstreamDoHConnectionsManager.moveToIdle(shared); + } +} + +void DoHConnectionToBackend::updateIO(IOState newState, FDMultiplexer::callbackfunc_t callback, bool noTTD) +{ + struct timeval now; + gettimeofday(&now, nullptr); + boost::optional ttd{boost::none}; + if (!noTTD) { + if (d_healthCheckQuery) { + ttd = getBackendHealthCheckTTD(now); + } + else if (newState == IOState::NeedRead) { + ttd = getBackendReadTTD(now); + } + else if (isFresh() && d_firstWrite) { + /* first write just after the non-blocking connect */ + ttd = getBackendConnectTTD(now); + } + else { + ttd = getBackendWriteTTD(now); + } + } + + auto shared = std::dynamic_pointer_cast(shared_from_this()); + if (shared) { + if (newState == IOState::NeedRead) { + d_ioState->update(newState, callback, shared, ttd); + } + else if (newState == IOState::NeedWrite) { + d_ioState->update(newState, callback, shared, ttd); + } + } +} + +void DoHConnectionToBackend::watchForRemoteHostClosingConnection() +{ + if (willBeReusable(false) && !d_healthCheckQuery) { + updateIO(IOState::NeedRead, handleReadableIOCallback, false); + } +} + +void DoHConnectionToBackend::addToIOState(IOState state, FDMultiplexer::callbackfunc_t callback) +{ + struct timeval now; + gettimeofday(&now, nullptr); + boost::optional ttd{boost::none}; + if (state == IOState::NeedRead) { + ttd = getBackendReadTTD(now); + } + else if (isFresh() && d_firstWrite == 0) { + /* first write just after the non-blocking connect */ + ttd = getBackendConnectTTD(now); + } + else { + ttd = getBackendWriteTTD(now); + } + + auto shared = std::dynamic_pointer_cast(shared_from_this()); + if (shared) { + if (state == IOState::NeedRead) { + d_ioState->add(state, callback, shared, ttd); + } + else if (state == IOState::NeedWrite) { + d_ioState->add(state, callback, shared, ttd); + } + } +} + +ssize_t DoHConnectionToBackend::send_callback(nghttp2_session* session, const uint8_t* data, size_t length, int flags, void* user_data) +{ + DoHConnectionToBackend* conn = reinterpret_cast(user_data); + bool bufferWasEmpty = conn->d_out.empty(); + if (!conn->d_proxyProtocolPayloadSent && !conn->d_proxyProtocolPayload.empty()) { + conn->d_out.insert(conn->d_out.end(), conn->d_proxyProtocolPayload.begin(), conn->d_proxyProtocolPayload.end()); + conn->d_proxyProtocolPayloadSent = true; + } + + conn->d_out.insert(conn->d_out.end(), data, data + length); + + if (bufferWasEmpty) { + try { + // cerr<<"in "<<__PRETTY_FUNCTION__<<" trying to write "<d_out.size()-conn->d_outPos<d_handler->tryWrite(conn->d_out, conn->d_outPos, conn->d_out.size()); + // cerr<<"got a "<<(int)state<<" state, "<d_out.size()-conn->d_outPos<<" bytes remaining"<d_firstWrite = false; + conn->d_out.clear(); + conn->d_outPos = 0; + conn->stopIO(); + if (!conn->isIdle()) { + conn->updateIO(IOState::NeedRead, handleReadableIOCallback); + } + else { + conn->watchForRemoteHostClosingConnection(); + } + } + else { + conn->updateIO(state, handleWritableIOCallback); + } + } + catch (const std::exception& e) { + vinfolog("Exception while trying to write (send) to HTTP backend connection: %s", e.what()); + conn->handleIOError(); + ++conn->d_ds->tcpDiedSendingQuery; + } + } + + return length; +} + +int DoHConnectionToBackend::on_frame_recv_callback(nghttp2_session* session, const nghttp2_frame* frame, void* user_data) +{ + DoHConnectionToBackend* conn = reinterpret_cast(user_data); + // cerr<<"Frame type is "<hd.type)<hd.type) { + case NGHTTP2_HEADERS: + cerr<<"got headers"<headers.cat == NGHTTP2_HCAT_RESPONSE) { + cerr<<"All headers received"<settings.niv<settings.niv; idx++) { + cerr<<"- "<settings.iv[idx].settings_id<<" "<settings.iv[idx].value<hd.type == NGHTTP2_GOAWAY) { + conn->d_connectionDied = true; + } + + /* is this the last frame for this stream? */ + else if ((frame->hd.type == NGHTTP2_HEADERS || frame->hd.type == NGHTTP2_DATA) && frame->hd.flags & NGHTTP2_FLAG_END_STREAM) { + auto stream = conn->d_currentStreams.find(frame->hd.stream_id); + if (stream != conn->d_currentStreams.end()) { + // cerr<<"Stream "<hd.stream_id<<" is now finished"<second.d_finished = true; + ++conn->d_queries; + + auto request = std::move(stream->second); + conn->d_currentStreams.erase(stream->first); + if (request.d_responseCode == 200U) { + conn->handleResponse(std::move(request)); + } + else { + vinfolog("HTTP response has a non-200 status code: %d", request.d_responseCode); + struct timeval now; + gettimeofday(&now, nullptr); + + conn->handleResponseError(std::move(request), now); + } + + if (conn->isIdle()) { + conn->stopIO(); + conn->watchForRemoteHostClosingConnection(); + } + } + else { + vinfolog("Stream %d NOT FOUND", frame->hd.stream_id); + conn->d_connectionDied = true; + ++conn->d_ds->tcpDiedReadingResponse; + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + } + + return 0; +} + +int DoHConnectionToBackend::on_data_chunk_recv_callback(nghttp2_session* session, uint8_t flags, int32_t stream_id, const uint8_t* data, size_t len, void* user_data) +{ + DoHConnectionToBackend* conn = reinterpret_cast(user_data); + // cerr<<"Got data of size "<d_currentStreams.find(stream_id); + if (stream == conn->d_currentStreams.end()) { + vinfolog("Unable to match the stream ID %d to a known one!", stream_id); + conn->d_connectionDied = true; + ++conn->d_ds->tcpDiedReadingResponse; + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + if (len > std::numeric_limits::max() || (std::numeric_limits::max() - stream->second.d_buffer.size()) < len) { + vinfolog("Data frame of size %d is too large for a DNS response (we already have %d)", len, stream->second.d_buffer.size()); + conn->d_connectionDied = true; + ++conn->d_ds->tcpDiedReadingResponse; + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + stream->second.d_buffer.insert(stream->second.d_buffer.end(), data, data + len); + if (stream->second.d_finished) { + // cerr<<"we now have the full response!"<(data), len)<second); + conn->d_currentStreams.erase(stream->first); + if (request.d_responseCode == 200U) { + conn->handleResponse(std::move(request)); + } + else { + vinfolog("HTTP response has a non-200 status code: %d", request.d_responseCode); + struct timeval now; + gettimeofday(&now, nullptr); + + conn->handleResponseError(std::move(request), now); + } + if (conn->isIdle()) { + conn->stopIO(); + conn->watchForRemoteHostClosingConnection(); + } + } + + return 0; +} + +int DoHConnectionToBackend::on_stream_close_callback(nghttp2_session* session, int32_t stream_id, uint32_t error_code, void* user_data) +{ + DoHConnectionToBackend* conn = reinterpret_cast(user_data); + + if (error_code == 0) { + return 0; + } + + // cerr << "Stream " << stream_id << " closed with error_code=" << error_code << endl; + conn->d_connectionDied = true; + ++conn->d_ds->tcpDiedReadingResponse; + + auto stream = conn->d_currentStreams.find(stream_id); + if (stream == conn->d_currentStreams.end()) { + /* we don't care, then */ + return 0; + } + + struct timeval now; + gettimeofday(&now, nullptr); + auto request = std::move(stream->second); + conn->d_currentStreams.erase(stream->first); + + // cerr<<"Query has "<d_ds->d_retries<d_ds->d_retries) { + // cerr<<"in "<<__PRETTY_FUNCTION__<<", looking for a connection to send a query of size "<d_mplexer, conn->d_ds, now, std::string(conn->d_proxyProtocolPayload)); + downstream->queueQuery(request.d_sender, std::move(request.d_query)); + } + else { + conn->handleResponseError(std::move(request), now); + } + + //cerr<<"we now have "<getConcurrentStreamsCount()<<" concurrent connections"<isIdle()) { + //cerr<<"stopping IO"<stopIO(); + conn->watchForRemoteHostClosingConnection(); + } + + return 0; +} + +int DoHConnectionToBackend::on_header_callback(nghttp2_session* session, const nghttp2_frame* frame, const uint8_t* name, size_t namelen, const uint8_t* value, size_t valuelen, uint8_t flags, void* user_data) +{ + DoHConnectionToBackend* conn = reinterpret_cast(user_data); + + const std::string status(":status"); + if (frame->hd.type == NGHTTP2_HEADERS && frame->headers.cat == NGHTTP2_HCAT_RESPONSE) { + //cerr<<"got header for "<hd.stream_id<<":"<(name), namelen)<(value), valuelen)<d_currentStreams.find(frame->hd.stream_id); + if (stream == conn->d_currentStreams.end()) { + vinfolog("Unable to match the stream ID %d to a known one!", frame->hd.stream_id); + conn->d_connectionDied = true; + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + try { + stream->second.d_responseCode = pdns_stou(std::string(reinterpret_cast(value), valuelen)); + } + catch (...) { + vinfolog("Error parsing the status header for stream ID %d", frame->hd.stream_id); + conn->d_connectionDied = true; + ++conn->d_ds->tcpDiedReadingResponse; + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + } + } + return 0; +} + +int DoHConnectionToBackend::on_error_callback(nghttp2_session* session, int lib_error_code, const char* msg, size_t len, void* user_data) +{ + vinfolog("Error in HTTP/2 connection: %s", std::string(msg, len)); + + DoHConnectionToBackend* conn = reinterpret_cast(user_data); + conn->d_connectionDied = true; + ++conn->d_ds->tcpDiedReadingResponse; + + return 0; +} + +DoHConnectionToBackend::DoHConnectionToBackend(const std::shared_ptr& ds, std::unique_ptr& mplexer, const struct timeval& now, std::string&& proxyProtocolPayload) : + ConnectionToBackend(ds, mplexer, now), d_proxyProtocolPayload(std::move(proxyProtocolPayload)) +{ + // inherit most of the stuff from the ConnectionToBackend() + d_ioState = make_unique(*d_mplexer, d_handler->getDescriptor()); + + nghttp2_session_callbacks* cbs = nullptr; + if (nghttp2_session_callbacks_new(&cbs) != 0) { + d_connectionDied = true; + ++d_ds->tcpDiedSendingQuery; + vinfolog("Unable to create a callback object for a new HTTP/2 session"); + return; + } + std::unique_ptr callbacks(cbs, nghttp2_session_callbacks_del); + cbs = nullptr; + + nghttp2_session_callbacks_set_send_callback(callbacks.get(), send_callback); + nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks.get(), on_frame_recv_callback); + nghttp2_session_callbacks_set_on_data_chunk_recv_callback(callbacks.get(), on_data_chunk_recv_callback); + nghttp2_session_callbacks_set_on_stream_close_callback(callbacks.get(), on_stream_close_callback); + nghttp2_session_callbacks_set_on_header_callback(callbacks.get(), on_header_callback); + nghttp2_session_callbacks_set_error_callback2(callbacks.get(), on_error_callback); + + nghttp2_session* sess = nullptr; + if (nghttp2_session_client_new(&sess, callbacks.get(), this) != 0) { + d_connectionDied = true; + ++d_ds->tcpDiedSendingQuery; + vinfolog("Coult not allocate a new HTTP/2 session"); + return; + } + + d_session = std::unique_ptr(sess, nghttp2_session_del); + sess = nullptr; + + callbacks.reset(); + + nghttp2_settings_entry iv[] = { + /* rfc7540 section-8.2.2: + "Advertising a SETTINGS_MAX_CONCURRENT_STREAMS value of zero disables + server push by preventing the server from creating the necessary + streams." + */ + {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, 0}, + {NGHTTP2_SETTINGS_ENABLE_PUSH, 0}, + /* we might want to make the initial window size configurable, but 16M is a large enough default */ + {NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, 16 * 1024 * 1024}}; + /* client 24 bytes magic string will be sent by nghttp2 library */ + int rv = nghttp2_submit_settings(d_session.get(), NGHTTP2_FLAG_NONE, iv, sizeof(iv) / sizeof(*iv)); + if (rv != 0) { + d_connectionDied = true; + ++d_ds->tcpDiedSendingQuery; + vinfolog("Could not submit SETTINGS: %s", nghttp2_strerror(rv)); + return; + } +} + +static void handleCrossProtocolQuery(int pipefd, FDMultiplexer::funcparam_t& param) +{ + auto threadData = boost::any_cast(param); + CrossProtocolQuery* tmp{nullptr}; + + ssize_t got = read(pipefd, &tmp, sizeof(tmp)); + if (got == 0) { + throw std::runtime_error("EOF while reading from the DoH cross-protocol pipe (" + std::to_string(pipefd) + ") in " + std::string(isNonBlocking(pipefd) ? "non-blocking" : "blocking") + " mode"); + } + else if (got == -1) { + if (errno == EAGAIN || errno == EINTR) { + return; + } + throw std::runtime_error("Error while reading from the DoH cross-protocol pipe (" + std::to_string(pipefd) + ") in " + std::string(isNonBlocking(pipefd) ? "non-blocking" : "blocking") + " mode:" + stringerror()); + } + else if (got != sizeof(tmp)) { + throw std::runtime_error("Partial read while reading from the DoH cross-protocol pipe (" + std::to_string(pipefd) + ") in " + std::string(isNonBlocking(pipefd) ? "non-blocking" : "blocking") + " mode"); + } + + try { + struct timeval now; + gettimeofday(&now, nullptr); + + std::shared_ptr tqs = tmp->getTCPQuerySender(); + auto query = std::move(tmp->query); + auto downstreamServer = std::move(tmp->downstream); + delete tmp; + tmp = nullptr; + + try { + auto downstream = t_downstreamDoHConnectionsManager.getConnectionToDownstream(threadData->mplexer, downstreamServer, now, std::move(query.d_proxyProtocolPayload)); + downstream->queueQuery(tqs, std::move(query)); + } + catch (...) { + tqs->notifyIOError(std::move(query.d_idstate), now); + } + } + catch (...) { + delete tmp; + tmp = nullptr; + } +} + +static void dohClientThread(int crossProtocolPipeFD) +{ + setThreadName("dnsdist/dohClie"); + + try { + DoHClientThreadData data; + data.mplexer->addReadFD(crossProtocolPipeFD, handleCrossProtocolQuery, &data); + + struct timeval now; + gettimeofday(&now, nullptr); + time_t lastTimeoutScan = now.tv_sec; + + for (;;) { + data.mplexer->run(&now); + + if (now.tv_sec > lastTimeoutScan) { + lastTimeoutScan = now.tv_sec; + + try { + t_downstreamDoHConnectionsManager.cleanupClosedConnections(now); + handleH2Timeouts(*data.mplexer, now); + + if (g_dohStatesDumpRequested > 0) { + /* just to keep things clean in the output, debug only */ + static std::mutex s_lock; + std::lock_guard lck(s_lock); + if (g_dohStatesDumpRequested > 0) { + /* no race here, we took the lock so it can only be increased in the meantime */ + --g_dohStatesDumpRequested; + errlog("Dumping the DoH client states, as requested:"); + data.mplexer->runForAllWatchedFDs([](bool isRead, int fd, const FDMultiplexer::funcparam_t& param, struct timeval ttd) { + struct timeval lnow; + gettimeofday(&lnow, nullptr); + if (ttd.tv_sec > 0) { + errlog("- Descriptor %d is in %s state, TTD in %d", fd, (isRead ? "read" : "write"), (ttd.tv_sec - lnow.tv_sec)); + } + else { + errlog("- Descriptor %d is in %s state, no TTD set", fd, (isRead ? "read" : "write")); + } + + if (param.type() == typeid(std::shared_ptr)) { + auto conn = boost::any_cast>(param); + errlog(" - %s", conn->toString()); + } + else if (param.type() == typeid(DoHClientThreadData*)) { + errlog(" - Worker thread pipe"); + } + }); + } + } + } + catch (const std::exception& e) { + errlog("Error in outgoing DoH thread: %s", e.what()); + } + } + } + } + catch (const std::exception& e) { + errlog("Fatal error in outgoing DoH thread: %s", e.what()); + } +} + +static bool select_next_proto_callback(unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen) +{ + if (nghttp2_select_next_protocol(out, outlen, in, inlen) <= 0) { + vinfolog("The remote DoH backend did not advertise " NGHTTP2_PROTO_VERSION_ID); + return false; + } + return true; +} + +#endif /* HAVE_NGHTTP2 */ + +struct DoHClientCollection::DoHWorkerThread +{ + DoHWorkerThread() + { + } + + DoHWorkerThread(int crossProtocolPipe) : + d_crossProtocolQueryPipe(crossProtocolPipe) + { + } + + DoHWorkerThread(DoHWorkerThread&& rhs) : + d_crossProtocolQueryPipe(rhs.d_crossProtocolQueryPipe) + { + rhs.d_crossProtocolQueryPipe = -1; + } + + DoHWorkerThread& operator=(DoHWorkerThread&& rhs) + { + if (d_crossProtocolQueryPipe != -1) { + close(d_crossProtocolQueryPipe); + } + + d_crossProtocolQueryPipe = rhs.d_crossProtocolQueryPipe; + rhs.d_crossProtocolQueryPipe = -1; + + return *this; + } + + DoHWorkerThread(const DoHWorkerThread& rhs) = delete; + DoHWorkerThread& operator=(const DoHWorkerThread&) = delete; + + ~DoHWorkerThread() + { + if (d_crossProtocolQueryPipe != -1) { + close(d_crossProtocolQueryPipe); + } + } + + int d_crossProtocolQueryPipe{-1}; +}; + +DoHClientCollection::DoHClientCollection(size_t numberOfThreads) : + d_clientThreads(numberOfThreads) +{ +} + +bool DoHClientCollection::passCrossProtocolQueryToThread(std::unique_ptr&& cpq) +{ + if (d_numberOfThreads == 0) { + throw std::runtime_error("No DoH worker thread yet"); + } + + uint64_t pos = d_pos++; + auto pipe = d_clientThreads.at(pos % d_numberOfThreads).d_crossProtocolQueryPipe; + auto tmp = cpq.release(); + + if (write(pipe, &tmp, sizeof(tmp)) != sizeof(tmp)) { + delete tmp; + ++g_stats.outgoingDoHQueryPipeFull; + tmp = nullptr; + return false; + } + + return true; +} + +void DoHClientCollection::addThread() +{ +#ifdef HAVE_NGHTTP2 + auto preparePipe = [](int fds[2], const std::string& type) -> bool { + if (pipe(fds) < 0) { + errlog("Error creating the DoH thread %s pipe: %s", type, stringerror()); + return false; + } + + if (!setNonBlocking(fds[0])) { + int err = errno; + close(fds[0]); + close(fds[1]); + errlog("Error setting the DoH thread %s pipe non-blocking: %s", type, stringerror(err)); + return false; + } + + if (!setNonBlocking(fds[1])) { + int err = errno; + close(fds[0]); + close(fds[1]); + errlog("Error setting the DoH thread %s pipe non-blocking: %s", type, stringerror(err)); + return false; + } + + if (g_tcpInternalPipeBufferSize > 0 && getPipeBufferSize(fds[0]) < g_tcpInternalPipeBufferSize) { + setPipeBufferSize(fds[0], g_tcpInternalPipeBufferSize); + } + + return true; + }; + + int crossProtocolFDs[2] = {-1, -1}; + if (!preparePipe(crossProtocolFDs, "cross-protocol")) { + return; + } + + vinfolog("Adding DoH Client thread"); + + { + std::lock_guard lock(d_mutex); + + if (d_numberOfThreads >= d_clientThreads.size()) { + vinfolog("Adding a new DoH client thread would exceed the vector size (%d/%d), skipping. Consider increasing the maximum amount of DoH client threads with setMaxDoHClientThreads() in the configuration.", d_numberOfThreads, d_clientThreads.size()); + close(crossProtocolFDs[0]); + close(crossProtocolFDs[1]); + return; + } + + /* from now on this side of the pipe will be managed by that object, + no need to worry about it */ + DoHWorkerThread worker(crossProtocolFDs[1]); + try { + std::thread t1(dohClientThread, crossProtocolFDs[0]); + t1.detach(); + } + catch (const std::runtime_error& e) { + /* the thread creation failed, don't leak */ + errlog("Error creating a DoH thread: %s", e.what()); + close(crossProtocolFDs[0]); + return; + } + + d_clientThreads.at(d_numberOfThreads) = std::move(worker); + ++d_numberOfThreads; + } +#else /* HAVE_NGHTTP2 */ + throw std::runtime_error("DoHClientCollection::addThread() called but nghttp2 support is not available"); +#endif /* HAVE_NGHTTP2 */ +} + +bool initDoHWorkers() +{ +#ifdef HAVE_NGHTTP2 + if (!g_outgoingDoHWorkerThreads) { + /* Unless the value has been set to 0 explicitly, always start at least one outgoing DoH worker thread, in case a DoH backend + is added at a later time. */ + g_outgoingDoHWorkerThreads = 1; + } + + if (g_outgoingDoHWorkerThreads && *g_outgoingDoHWorkerThreads > 0) { + g_dohClientThreads = std::make_unique(*g_outgoingDoHWorkerThreads); + for (size_t idx = 0; idx < *g_outgoingDoHWorkerThreads; idx++) { + g_dohClientThreads->addThread(); + } + } + return true; +#else + return false; +#endif /* HAVE_NGHTTP2 */ +} + +bool setupDoHClientProtocolNegotiation(std::shared_ptr& ctx) +{ + if (ctx == nullptr) { + return false; + } +#ifdef HAVE_NGHTTP2 + /* we want to set the ALPN to h2, if only to mitigate the ALPACA attack */ + const std::vector> h2Alpns = {{'h', '2'}}; + ctx->setALPNProtos(h2Alpns); + ctx->setNextProtocolSelectCallback(select_next_proto_callback); + return true; +#else /* HAVE_NGHTTP2 */ + return false; +#endif /* HAVE_NGHTTP2 */ +} + +bool sendH2Query(const std::shared_ptr& ds, std::unique_ptr& mplexer, std::shared_ptr& sender, InternalQuery&& query, bool healthCheck) +{ +#ifdef HAVE_NGHTTP2 + struct timeval now; + gettimeofday(&now, nullptr); + + if (healthCheck) { + /* always do health-checks over a new connection */ + auto newConnection = std::make_shared(ds, mplexer, now, std::move(query.d_proxyProtocolPayload)); + newConnection->setHealthCheck(healthCheck); + newConnection->queueQuery(sender, std::move(query)); + } + else { + auto connection = t_downstreamDoHConnectionsManager.getConnectionToDownstream(mplexer, ds, now, std::move(query.d_proxyProtocolPayload)); + connection->queueQuery(sender, std::move(query)); + } + + return true; +#else /* HAVE_NGHTTP2 */ + return false; +#endif /* HAVE_NGHTTP2 */ +} + +size_t clearH2Connections() +{ + size_t cleared = 0; +#ifdef HAVE_NGHTTP2 + cleared = t_downstreamDoHConnectionsManager.clear(); +#endif /* HAVE_NGHTTP2 */ + return cleared; +} + +size_t handleH2Timeouts(FDMultiplexer& mplexer, const struct timeval& now) +{ + size_t got = 0; +#ifdef HAVE_NGHTTP2 + auto expiredReadConns = mplexer.getTimeouts(now, false); + for (const auto& cbData : expiredReadConns) { + if (cbData.second.type() == typeid(std::shared_ptr)) { + auto conn = boost::any_cast>(cbData.second); + vinfolog("Timeout (read) from remote DoH backend %s", conn->getBackendName()); + conn->handleTimeout(now, false); + ++got; + } + } + + auto expiredWriteConns = mplexer.getTimeouts(now, true); + for (const auto& cbData : expiredWriteConns) { + if (cbData.second.type() == typeid(std::shared_ptr)) { + auto conn = boost::any_cast>(cbData.second); + vinfolog("Timeout (write) from remote DoH backend %s", conn->getBackendName()); + conn->handleTimeout(now, true); + ++got; + } + } +#endif /* HAVE_NGHTTP2 */ + return got; +} + +void setDoHDownstreamCleanupInterval(uint16_t max) +{ +#ifdef HAVE_NGHTTP2 + DownstreamDoHConnectionsManager::setCleanupInterval(max); +#endif /* HAVE_NGHTTP2 */ +} + +void setDoHDownstreamMaxIdleTime(uint16_t max) +{ +#ifdef HAVE_NGHTTP2 + DownstreamDoHConnectionsManager::setMaxIdleTime(max); +#endif /* HAVE_NGHTTP2 */ +} + +void setDoHDownstreamMaxIdleConnectionsPerBackend(size_t max) +{ +#ifdef HAVE_NGHTTP2 + DownstreamDoHConnectionsManager::setMaxIdleConnectionsPerDownstream(max); +#endif /* HAVE_NGHTTP2 */ +} diff --git a/pdns/dnsdistdist/dnsdist-nghttp2.hh b/pdns/dnsdistdist/dnsdist-nghttp2.hh new file mode 100644 index 000000000000..6e38f28cc383 --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-nghttp2.hh @@ -0,0 +1,75 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include +#include + +#include "dnsdist-tcp.hh" +#include "stat_t.hh" + +struct CrossProtocolQuery; + +class DoHClientCollection +{ +public: + DoHClientCollection(size_t numberOfThreads); + + uint64_t getThreadsCount() const + { + return d_numberOfThreads; + } + + bool passCrossProtocolQueryToThread(std::unique_ptr&& cpq); + void addThread(); + +private: + struct DoHWorkerThread; + + std::mutex d_mutex; + /* we only alter that vector at configuration time, and then + it is never modified at runtime, so we don't take a lock + after the configuration phase */ + std::vector d_clientThreads; + pdns::stat_t d_pos{0}; + uint64_t d_numberOfThreads{0}; +}; + +extern std::unique_ptr g_dohClientThreads; +extern std::atomic g_dohStatesDumpRequested; +extern std::optional g_outgoingDoHWorkerThreads; + +class TLSCtx; + +bool initDoHWorkers(); +bool setupDoHClientProtocolNegotiation(std::shared_ptr& ctx); + +/* opens a new HTTP/2 connection to the supplied backend (attached to the supplied multiplexer), sends the query, + waits for the response to come back or an error to occur then notifies the sender, closing the connection. */ +bool sendH2Query(const std::shared_ptr& ds, std::unique_ptr& mplexer, std::shared_ptr& sender, InternalQuery&& query, bool healthCheck); +size_t handleH2Timeouts(FDMultiplexer& mplexer, const struct timeval& now); +size_t clearH2Connections(); + +void setDoHDownstreamCleanupInterval(uint16_t max); +void setDoHDownstreamMaxIdleTime(uint16_t max); +void setDoHDownstreamMaxIdleConnectionsPerBackend(size_t max); diff --git a/pdns/dnsdistdist/dnsdist-protocols.cc b/pdns/dnsdistdist/dnsdist-protocols.cc deleted file mode 100644 index 5c351c153ada..000000000000 --- a/pdns/dnsdistdist/dnsdist-protocols.cc +++ /dev/null @@ -1,31 +0,0 @@ -/* - * This file is part of PowerDNS or dnsdist. - * Copyright -- PowerDNS.COM B.V. and its contributors - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * In addition, for the avoidance of any doubt, permission is granted to - * link this program with OpenSSL and to (re)distribute the binaries - * produced as the result of such linking. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ -#include "dnsdist-protocols.hh" - -namespace dnsdist -{ -const std::string& ProtocolToString(Protocol proto) -{ - static const std::vector values = {"Do53 UDP", "Do53 TCP", "DNSCrypt UDP", "DNSCrypt TCP", "DNS over TLS", "DNS over HTTPS"}; - return values.at(static_cast(proto)); -} -} diff --git a/pdns/dnsdistdist/dnsdist-protocols.cc b/pdns/dnsdistdist/dnsdist-protocols.cc new file mode 120000 index 000000000000..eb08cd3869eb --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-protocols.cc @@ -0,0 +1 @@ +../dnsdist-protocols.cc \ No newline at end of file diff --git a/pdns/dnsdistdist/dnsdist-rules.hh b/pdns/dnsdistdist/dnsdist-rules.hh index 6e0e5d92322d..dda702e4356b 100644 --- a/pdns/dnsdistdist/dnsdist-rules.hh +++ b/pdns/dnsdistdist/dnsdist-rules.hh @@ -1082,6 +1082,28 @@ private: std::string d_poolname; }; +class PoolOutstandingRule : public DNSRule +{ +public: + PoolOutstandingRule(const std::string& poolname, const size_t limit) : d_pools(&g_pools), d_poolname(poolname), d_limit(limit) + { + } + + bool matches(const DNSQuestion* dq) const override + { + return (getPool(*d_pools, d_poolname)->poolLoad()) > d_limit; + } + + string toString() const override + { + return "pool '" + d_poolname + "' outstanding > " + std::to_string(d_limit); + } +private: + mutable LocalStateHolder d_pools; + std::string d_poolname; + size_t d_limit; +}; + class KeyValueStoreLookupRule: public DNSRule { public: diff --git a/pdns/dnsdistdist/dnsdist-session-cache.cc b/pdns/dnsdistdist/dnsdist-session-cache.cc index 67de19536048..b227efd2c6b6 100644 --- a/pdns/dnsdistdist/dnsdist-session-cache.cc +++ b/pdns/dnsdistdist/dnsdist-session-cache.cc @@ -27,41 +27,43 @@ time_t TLSSessionCache::s_cleanupDelay{60}; time_t TLSSessionCache::s_sessionValidity{600}; uint16_t TLSSessionCache::s_maxSessionsPerBackend{20}; -void TLSSessionCache::cleanup(time_t now, const std::lock_guard& lock) +void TLSSessionCache::cleanup(time_t now, LockGuardedHolder& data) { time_t cutOff = now + s_sessionValidity; - for (auto it = d_sessions.begin(); it != d_sessions.end();) { + for (auto it = data->d_sessions.begin(); it != data->d_sessions.end();) { if (it->second.d_lastUsed > cutOff || it->second.d_sessions.size() == 0) { - it = d_sessions.erase(it); + it = data->d_sessions.erase(it); } else { ++it; } } - d_nextCleanup = now + s_cleanupDelay; + data->d_nextCleanup = now + s_cleanupDelay; } -void TLSSessionCache::putSession(const boost::uuids::uuid& backendID, time_t now, std::unique_ptr&& session) +void TLSSessionCache::putSessions(const boost::uuids::uuid& backendID, time_t now, std::vector>&& sessions) { - std::lock_guard lock(d_lock); - if (d_nextCleanup == 0 || now > d_nextCleanup) { - cleanup(now, lock); + auto data = d_data.lock(); + if (data->d_nextCleanup == 0 || now > data->d_nextCleanup) { + cleanup(now, data); } - auto& entry = d_sessions[backendID]; - if (entry.d_sessions.size() >= s_maxSessionsPerBackend) { - entry.d_sessions.pop_back(); + for (auto& session : sessions) { + auto& entry = data->d_sessions[backendID]; + if (entry.d_sessions.size() >= s_maxSessionsPerBackend) { + entry.d_sessions.pop_back(); + } + entry.d_sessions.push_front(std::move(session)); } - entry.d_sessions.push_front(std::move(session)); } std::unique_ptr TLSSessionCache::getSession(const boost::uuids::uuid& backendID, time_t now) { - std::lock_guard lock(d_lock); - auto it = d_sessions.find(backendID); - if (it == d_sessions.end()) { + auto data = d_data.lock(); + auto it = data->d_sessions.find(backendID); + if (it == data->d_sessions.end()) { return nullptr; } diff --git a/pdns/dnsdistdist/dnsdist-session-cache.hh b/pdns/dnsdistdist/dnsdist-session-cache.hh index 6cae14034bbb..47d76bc47807 100644 --- a/pdns/dnsdistdist/dnsdist-session-cache.hh +++ b/pdns/dnsdistdist/dnsdist-session-cache.hh @@ -24,6 +24,7 @@ #include #include +#include "lock.hh" #include "tcpiohandler.hh" #include "uuid-utils.hh" @@ -33,9 +34,8 @@ public: TLSSessionCache() { } - void cleanup(time_t now, const std::lock_guard& lock); - void putSession(const boost::uuids::uuid& backendID, time_t now, std::unique_ptr&& session); + void putSessions(const boost::uuids::uuid& backendID, time_t now, std::vector>&& sessions); std::unique_ptr getSession(const boost::uuids::uuid& backendID, time_t now); static void setCleanupDelay(time_t delay) @@ -64,10 +64,15 @@ private: time_t d_lastUsed{0}; }; - std::map d_sessions; - // do we need to shard this? - std::mutex d_lock; - time_t d_nextCleanup{0}; + struct CacheData + { + // do we need to shard this? + std::map d_sessions; + time_t d_nextCleanup{0}; + }; + LockGuarded d_data; + + void cleanup(time_t now, LockGuardedHolder& data); }; extern TLSSessionCache g_sessionCache; diff --git a/pdns/dnsdistdist/dnsdist-svc.cc b/pdns/dnsdistdist/dnsdist-svc.cc new file mode 100644 index 000000000000..ffd42fd64943 --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-svc.cc @@ -0,0 +1,133 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "dnsdist-svc.hh" +#include "dnswriter.hh" +#include "svc-records.hh" + +bool generateSVCPayload(std::vector& payload, uint16_t priority, const DNSName& target, const std::set& mandatoryParams, const std::vector& alpns, bool noDefaultAlpn, std::optional port, const std::string& ech, const std::vector& ipv4hints, const std::vector& ipv6hints, const std::vector>& additionalParams) +{ + // this is an _ordered_ set and the comparison operator is properly defined, + // so the parameters will be ordered as defined in the RFC + std::set params; + + if (!mandatoryParams.empty()) { + std::set mandatoryKeys; + for (const auto& entry : mandatoryParams) { + mandatoryKeys.insert(static_cast(entry)); + } + params.insert({SvcParam::SvcParamKey::mandatory, std::move(mandatoryKeys)}); + } + + if (!alpns.empty()) { + params.insert({SvcParam::SvcParamKey::alpn, std::vector(alpns)}); + } + + if (noDefaultAlpn) { + params.insert({SvcParam::SvcParamKey::no_default_alpn}); + } + + if (port) { + params.insert({SvcParam::SvcParamKey::port, *port}); + } + + if (!ipv4hints.empty()) { + params.insert({SvcParam::SvcParamKey::ipv4hint, std::vector(ipv4hints)}); + } + + if (!ech.empty()) { + params.insert({SvcParam::SvcParamKey::ech, ech}); + } + + if (!ipv6hints.empty()) { + params.insert({SvcParam::SvcParamKey::ipv6hint, std::vector(ipv6hints)}); + } + + for (const auto& param : additionalParams) { + params.insert({static_cast(param.first), param.second}); + } + + if (priority == 0 && params.size() != 0) { + return false; + } + + payload.clear(); + /* we will remove the header, question and record header parts later */ + DNSPacketWriter pw(payload, g_rootdnsname, QType::A, QClass::IN, 0); + pw.startRecord(g_rootdnsname, QType::A, 60, QClass::IN, DNSResourceRecord::ANSWER, false); + size_t offset = pw.size(); + pw.xfr16BitInt(priority); + pw.xfrName(target, false, true); + pw.xfrSvcParamKeyVals(params); + pw.commit(); + + if (payload.size() <= offset) { + return false; + } + + payload.erase(payload.begin(), payload.begin() + offset); + return true; +} + +bool generateSVCPayload(std::vector& payload, const SVCRecordParameters& parameters) +{ + return generateSVCPayload(payload, parameters.priority, parameters.target, parameters.mandatoryParams, parameters.alpns, parameters.noDefaultAlpn, parameters.port, parameters.ech, parameters.ipv4hints, parameters.ipv6hints, parameters.additionalParams); +} + +struct SVCRecordParameters parseSVCParameters(const svcParamsLua_t& params) +{ + struct SVCRecordParameters parameters; + for (const auto& p : params) { + if (p.first == "mandatory") { + for (auto const& entry : boost::get>>(p.second)) { + parameters.mandatoryParams.insert(SvcParam::keyFromString(entry.second)); + } + } + else if (p.first == "alpn") { + for (auto const& entry : boost::get>>(p.second)) { + parameters.alpns.push_back(entry.second); + } + } + else if (p.first == "noDefaultAlpn") { + parameters.noDefaultAlpn = boost::get(p.second); + } + else if (p.first == "port") { + parameters.port = boost::get(p.second); + } + else if (p.first == "ipv4hint") { + for (auto const& entry : boost::get>>(p.second)) { + parameters.ipv4hints.push_back(ComboAddress(entry.second)); + } + } + else if (p.first == "ech") { + parameters.ech = boost::get(p.second); + } + else if (p.first == "ipv6hint") { + for (auto const& entry : boost::get>>(p.second)) { + parameters.ipv6hints.push_back(ComboAddress(entry.second)); + } + } + else { + parameters.additionalParams.push_back({SvcParam::keyFromString(p.first), boost::get(p.second)}); + } + } + return parameters; +} diff --git a/pdns/dnsdistdist/dnsdist-svc.hh b/pdns/dnsdistdist/dnsdist-svc.hh new file mode 100644 index 000000000000..d0a1a8c961b3 --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-svc.hh @@ -0,0 +1,66 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "dnsname.hh" +#include "iputils.hh" + +struct SVCRecordParameters +{ + SVCRecordParameters() + { + } + + std::set mandatoryParams; + std::vector alpns; + std::vector ipv4hints; + std::vector ipv6hints; + std::vector> additionalParams; + std::string ech; + DNSName target; + std::optional port{std::nullopt}; + uint16_t priority{0}; + bool noDefaultAlpn{false}; +}; + +typedef std::unordered_map< + std::string, + boost::variant< + uint16_t, + bool, + std::string, + std::vector>, + std::vector>>> + svcParamsLua_t; + +struct SVCRecordParameters parseSVCParameters(const svcParamsLua_t& params); + +bool generateSVCPayload(std::vector& payload, uint16_t priority, const DNSName& target, const std::set& mandatoryParams, const std::vector& alpns, bool noDefaultAlpn, std::optional port, const std::string& ech, const std::vector& ipv4hints, const std::vector& ipv6hints, const std::vector>& additionalParams); + +bool generateSVCPayload(std::vector& payload, const SVCRecordParameters& parameters); diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc index 99945ffaf3f1..7d3158cc215d 100644 --- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc +++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc @@ -5,7 +5,9 @@ #include "dnsparser.hh" -TCPConnectionToBackend::~TCPConnectionToBackend() +thread_local DownstreamTCPConnectionsManager t_downstreamTCPConnectionsManager; + +ConnectionToBackend::~ConnectionToBackend() { if (d_ds && d_handler) { --d_ds->tcpCurrentConnections; @@ -17,9 +19,9 @@ TCPConnectionToBackend::~TCPConnectionToBackend() ++d_ds->tlsResumptions; } try { - auto session = d_handler->getTLSSession(); - if (session) { - g_sessionCache.putSession(d_ds->getID(), now.tv_sec, std::move(session)); + auto sessions = d_handler->getTLSSessions(); + if (!sessions.empty()) { + g_sessionCache.putSessions(d_ds->getID(), now.tv_sec, std::move(sessions)); } } catch (const std::exception& e) { @@ -27,10 +29,106 @@ TCPConnectionToBackend::~TCPConnectionToBackend() } } auto diff = now - d_connectionStartTime; + // cerr<<"connection to backend terminated after "<updateTCPMetrics(d_queries, diff.tv_sec * 1000 + diff.tv_usec / 1000); } } +bool ConnectionToBackend::reconnect() +{ + std::unique_ptr tlsSession{nullptr}; + if (d_handler) { + DEBUGLOG("closing socket "<getDescriptor()); + if (d_handler->isTLS()) { + if (d_handler->hasTLSSessionBeenResumed()) { + ++d_ds->tlsResumptions; + } + try { + auto sessions = d_handler->getTLSSessions(); + if (!sessions.empty()) { + tlsSession = std::move(sessions.back()); + sessions.pop_back(); + if (!sessions.empty()) { + g_sessionCache.putSessions(d_ds->getID(), time(nullptr), std::move(sessions)); + } + } + } + catch (const std::exception& e) { + vinfolog("Unable to get a TLS session to resume: %s", e.what()); + } + } + d_handler->close(); + d_ioState.reset(); + d_handler.reset(); + --d_ds->tcpCurrentConnections; + } + + d_fresh = true; + d_highestStreamID = 0; + d_proxyProtocolPayloadSent = false; + + do { + vinfolog("TCP connecting to downstream %s (%d)", d_ds->getNameWithAddr(), d_downstreamFailures); + DEBUGLOG("Opening TCP connection to backend "<getNameWithAddr()); + ++d_ds->tcpNewConnections; + try { + auto socket = std::make_unique(d_ds->remote.sin4.sin_family, SOCK_STREAM, 0); + DEBUGLOG("result of socket() is "<getHandle()); + + if (!IsAnyAddress(d_ds->sourceAddr)) { + SSetsockopt(socket->getHandle(), SOL_SOCKET, SO_REUSEADDR, 1); +#ifdef IP_BIND_ADDRESS_NO_PORT + if (d_ds->ipBindAddrNoPort) { + SSetsockopt(socket->getHandle(), SOL_IP, IP_BIND_ADDRESS_NO_PORT, 1); + } +#endif +#ifdef SO_BINDTODEVICE + if (!d_ds->sourceItfName.empty()) { + int res = setsockopt(socket->getHandle(), SOL_SOCKET, SO_BINDTODEVICE, d_ds->sourceItfName.c_str(), d_ds->sourceItfName.length()); + if (res != 0) { + vinfolog("Error setting up the interface on backend TCP socket '%s': %s", d_ds->getNameWithAddr(), stringerror()); + } + } +#endif + socket->bind(d_ds->sourceAddr, false); + } + socket->setNonBlocking(); + + gettimeofday(&d_connectionStartTime, nullptr); + auto handler = std::make_unique(d_ds->d_tlsSubjectName, socket->releaseHandle(), timeval{0,0}, d_ds->d_tlsCtx, d_connectionStartTime.tv_sec); + if (!tlsSession && d_ds->d_tlsCtx) { + tlsSession = g_sessionCache.getSession(d_ds->getID(), d_connectionStartTime.tv_sec); + } + if (tlsSession) { + handler->setTLSSession(tlsSession); + } + handler->tryConnect(d_ds->tcpFastOpen && isFastOpenEnabled(), d_ds->remote); + d_queries = 0; + + d_handler = std::move(handler); + d_ds->incCurrentConnectionsCount(); + return true; + } + catch (const std::runtime_error& e) { + vinfolog("Connection to downstream server %s failed: %s", d_ds->getName(), e.what()); + d_downstreamFailures++; + if (d_downstreamFailures >= d_ds->d_retries) { + throw; + } + } + } + while (d_downstreamFailures < d_ds->d_retries); + + return false; +} + +TCPConnectionToBackend::~TCPConnectionToBackend() +{ + if (d_ds && !d_pendingResponses.empty()) { + d_ds->outstanding -= d_pendingResponses.size(); + } +} + void TCPConnectionToBackend::release() { d_ds->outstanding -= d_pendingResponses.size(); @@ -38,15 +136,61 @@ void TCPConnectionToBackend::release() d_pendingResponses.clear(); d_pendingQueries.clear(); - d_sender.reset(); if (d_ioState) { d_ioState.reset(); } } +static void editPayloadID(PacketBuffer& payload, uint16_t newId, size_t proxyProtocolPayloadSize, bool sizePrepended) +{ + /* we cannot do a direct cast as the alignment might be off (the size of the payload might have been prepended, which is bad enough, + but we might also have a proxy protocol payload */ + size_t startOfHeaderOffset = (sizePrepended ? sizeof(uint16_t) : 0) + proxyProtocolPayloadSize; + if (payload.size() < startOfHeaderOffset + sizeof(dnsheader)) { + throw std::runtime_error("Invalid buffer for outgoing TCP query (size " + std::to_string(payload.size())); + } + uint16_t id = htons(newId); + memcpy(&payload.at(startOfHeaderOffset), &id, sizeof(id)); +} + +enum class QueryState : uint8_t { + hasSizePrepended, + noSize +}; + +enum class ConnectionState : uint8_t { + needProxy, + proxySent +}; + +static void prepareQueryForSending(TCPQuery& query, uint16_t id, QueryState queryState, ConnectionState connectionState) +{ + if (connectionState == ConnectionState::needProxy) { + if (query.d_proxyProtocolPayload.size() > 0 && !query.d_proxyProtocolPayloadAdded) { + query.d_buffer.insert(query.d_buffer.begin(), query.d_proxyProtocolPayload.begin(), query.d_proxyProtocolPayload.end()); + query.d_proxyProtocolPayloadAdded = true; + } + } + else if (connectionState == ConnectionState::proxySent) { + if (query.d_proxyProtocolPayloadAdded) { + if (query.d_buffer.size() < query.d_proxyProtocolPayload.size()) { + throw std::runtime_error("Trying to remove a proxy protocol payload of size " + std::to_string(query.d_proxyProtocolPayload.size()) + " from a buffer of size " + std::to_string(query.d_buffer.size())); + } + query.d_buffer.erase(query.d_buffer.begin(), query.d_buffer.begin() + query.d_proxyProtocolPayload.size()); + query.d_proxyProtocolPayloadAdded = false; + } + } + + editPayloadID(query.d_buffer, id, query.d_proxyProtocolPayloadAdded ? query.d_proxyProtocolPayload.size() : 0, true); +} + IOState TCPConnectionToBackend::queueNextQuery(std::shared_ptr& conn) { conn->d_currentQuery = std::move(conn->d_pendingQueries.front()); + + uint16_t id = conn->d_highestStreamID; + prepareQueryForSending(conn->d_currentQuery.d_query, id, QueryState::hasSizePrepended, conn->needProxyProtocolPayload() ? ConnectionState::needProxy : ConnectionState::proxySent); + conn->d_pendingQueries.pop_front(); conn->d_state = State::sendingQueryToBackend; conn->d_currentPos = 0; @@ -58,7 +202,7 @@ IOState TCPConnectionToBackend::sendQuery(std::shared_ptrgetDS()->getName()<<" over FD "<d_handler->getDescriptor()); - IOState state = conn->d_handler->tryWrite(conn->d_currentQuery.d_buffer, conn->d_currentPos, conn->d_currentQuery.d_buffer.size()); + IOState state = conn->d_handler->tryWrite(conn->d_currentQuery.d_query.d_buffer, conn->d_currentPos, conn->d_currentQuery.d_query.d_buffer.size()); if (state != IOState::Done) { return state; @@ -66,17 +210,22 @@ IOState TCPConnectionToBackend::sendQuery(std::shared_ptrd_currentQuery.d_proxyProtocolPayloadAdded) { + if (conn->d_currentQuery.d_query.d_proxyProtocolPayloadAdded) { conn->d_proxyProtocolPayloadSent = true; } - conn->incQueries(); + ++conn->d_queries; conn->d_currentPos = 0; - DEBUGLOG("adding a pending response for ID "<d_currentQuery.d_idstate.origID)<<" and QNAME "<d_currentQuery.d_idstate.qname); - conn->d_pendingResponses[ntohs(conn->d_currentQuery.d_idstate.origID)] = std::move(conn->d_currentQuery); - conn->d_currentQuery.d_buffer.clear(); - - ++conn->d_ds->outstanding; + DEBUGLOG("adding a pending response for ID "<d_highestStreamID<<" and QNAME "<d_currentQuery.d_query.d_idstate.qname); + auto res = conn->d_pendingResponses.insert({conn->d_highestStreamID, std::move(conn->d_currentQuery)}); + /* if there was already a pending response with that ID, we messed up and we don't expect more + than one response */ + if (res.second) { + ++conn->d_ds->outstanding; + } + ++conn->d_highestStreamID; + conn->d_currentQuery.d_sender.reset(); + conn->d_currentQuery.d_query.d_buffer.clear(); return state; } @@ -144,7 +293,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr& c iostate = conn->handleResponse(conn, now); } catch (const std::exception& e) { - vinfolog("Got an exception while handling TCP response from %s (client is %s): %s", conn->d_ds ? conn->d_ds->getName() : "unknown", conn->d_currentQuery.d_idstate.origRemote.toStringWithPort(), e.what()); + vinfolog("Got an exception while handling TCP response from %s (client is %s): %s", conn->d_ds ? conn->d_ds->getName() : "unknown", conn->d_currentQuery.d_query.d_idstate.origRemote.toStringWithPort(), e.what()); ioGuard.release(); conn->release(); return; @@ -165,7 +314,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr& c but it might also be a real IO error or something else. Let's just drop the connection */ - vinfolog("Got an exception while handling (%s backend) TCP query from %s: %s", (conn->d_state == State::sendingQueryToBackend ? "writing to" : "reading from"), conn->d_currentQuery.d_idstate.origRemote.toStringWithPort(), e.what()); + vinfolog("Got an exception while handling (%s backend) TCP query from %s: %s", (conn->d_state == State::sendingQueryToBackend ? "writing to" : "reading from"), conn->d_currentQuery.d_query.d_idstate.origRemote.toStringWithPort(), e.what()); if (conn->d_state == State::sendingQueryToBackend) { ++conn->d_ds->tcpDiedSendingQuery; @@ -186,9 +335,9 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr& c if (connectionDied) { - DEBUGLOG("connection died, number of failures is "<d_downstreamFailures<<", retries is "<d_ds->retries); + DEBUGLOG("connection died, number of failures is "<d_downstreamFailures<<", retries is "<d_ds->d_retries); - if (conn->d_downstreamFailures < conn->d_ds->retries) { + if (conn->d_downstreamFailures < conn->d_ds->d_retries) { conn->d_ioState.reset(); ioGuard.release(); @@ -198,14 +347,22 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr& c conn->d_ioState = make_unique(*conn->d_mplexer, conn->d_handler->getDescriptor()); /* we need to resend the queries that were in flight, if any */ + if (conn->d_state == State::sendingQueryToBackend) { + /* we need to edit this query so it has the correct ID */ + auto query = std::move(conn->d_currentQuery); + uint16_t id = conn->d_highestStreamID; + prepareQueryForSending(query.d_query, id, QueryState::hasSizePrepended, ConnectionState::needProxy); + conn->d_currentQuery = std::move(query); + } + for (auto& pending : conn->d_pendingResponses) { --conn->d_ds->outstanding; - if (pending.second.isXFR() && pending.second.d_xfrStarted) { + if (pending.second.d_query.isXFR() && pending.second.d_query.d_xfrStarted) { /* this one can't be restarted, sorry */ DEBUGLOG("A XFR for which a response has already been sent cannot be restarted"); try { - conn->d_sender->notifyIOError(std::move(pending.second.d_idstate), now); + pending.second.d_sender->notifyIOError(std::move(pending.second.d_query.d_idstate), now); } catch (const std::exception& e) { vinfolog("Got an exception while notifying: %s", e.what()); @@ -221,8 +378,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr& c conn->d_pendingResponses.clear(); conn->d_currentPos = 0; - if (conn->d_state == State::doingHandshake || - conn->d_state == State::sendingQueryToBackend) { + if (conn->d_state == State::sendingQueryToBackend) { iostate = IOState::NeedWrite; // resume sending query } @@ -234,11 +390,6 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr& c iostate = queueNextQuery(conn); } - if (conn->needProxyProtocolPayload() && !conn->d_currentQuery.d_proxyProtocolPayloadAdded && !conn->d_currentQuery.d_proxyProtocolPayload.empty()) { - conn->d_currentQuery.d_buffer.insert(conn->d_currentQuery.d_buffer.begin(), conn->d_currentQuery.d_proxyProtocolPayload.begin(), conn->d_currentQuery.d_proxyProtocolPayload.end()); - conn->d_currentQuery.d_proxyProtocolPayloadAdded = true; - } - reconnected = true; connectionDied = false; } @@ -291,122 +442,39 @@ void TCPConnectionToBackend::handleIOCallback(int fd, FDMultiplexer::funcparam_t } struct timeval now; - gettimeofday(&now, 0); + gettimeofday(&now, nullptr); handleIO(conn, now); } void TCPConnectionToBackend::queueQuery(std::shared_ptr& sender, TCPQuery&& query) { - if (!d_sender) { - d_sender = sender; + if (!d_ioState) { d_ioState = make_unique(*d_mplexer, d_handler->getDescriptor()); } - else if (d_sender != sender) { - throw std::runtime_error("Assigning a query from a different client to an existing backend connection with pending queries"); - } - // if we are not already sending a query or in the middle of reading a response (so idle or doingHandshake), + // if we are not already sending a query or in the middle of reading a response (so idle), // start sending the query if (d_state == State::idle || d_state == State::waitingForResponseFromBackend) { - DEBUGLOG("Sending new query to backend right away"); + DEBUGLOG("Sending new query to backend right away, with ID "<(shared_from_this()); handleIO(shared, now); } else { DEBUGLOG("Adding new query to the queue because we are in state "<<(int)d_state); // store query in the list of queries to send - d_pendingQueries.push_back(std::move(query)); - } -} - -bool TCPConnectionToBackend::reconnect() -{ - std::unique_ptr tlsSession{nullptr}; - if (d_handler) { - DEBUGLOG("closing socket "<getDescriptor()); - if (d_handler->isTLS()) { - if (d_handler->hasTLSSessionBeenResumed()) { - ++d_ds->tlsResumptions; - } - try { - tlsSession = d_handler->getTLSSession(); - } - catch (const std::exception& e) { - vinfolog("Unable to get a TLS session to resume: %s", e.what()); - } - } - d_handler->close(); - d_ioState.reset(); - --d_ds->tcpCurrentConnections; - } - - d_fresh = true; - d_proxyProtocolPayloadSent = false; - - do { - vinfolog("TCP connecting to downstream %s (%d)", d_ds->getNameWithAddr(), d_downstreamFailures); - DEBUGLOG("Opening TCP connection to backend "<getNameWithAddr()); - ++d_ds->tcpNewConnections; - try { - auto socket = std::make_unique(d_ds->remote.sin4.sin_family, SOCK_STREAM, 0); - DEBUGLOG("result of socket() is "<getHandle()); - - if (!IsAnyAddress(d_ds->sourceAddr)) { - SSetsockopt(socket->getHandle(), SOL_SOCKET, SO_REUSEADDR, 1); -#ifdef IP_BIND_ADDRESS_NO_PORT - if (d_ds->ipBindAddrNoPort) { - SSetsockopt(socket->getHandle(), SOL_IP, IP_BIND_ADDRESS_NO_PORT, 1); - } -#endif -#ifdef SO_BINDTODEVICE - if (!d_ds->sourceItfName.empty()) { - int res = setsockopt(socket->getHandle(), SOL_SOCKET, SO_BINDTODEVICE, d_ds->sourceItfName.c_str(), d_ds->sourceItfName.length()); - if (res != 0) { - vinfolog("Error setting up the interface on backend TCP socket '%s': %s", d_ds->getNameWithAddr(), stringerror()); - } - } -#endif - socket->bind(d_ds->sourceAddr, false); - } - socket->setNonBlocking(); - - gettimeofday(&d_connectionStartTime, nullptr); - auto handler = std::make_unique(d_ds->d_tlsSubjectName, socket->releaseHandle(), timeval{0,0}, d_ds->d_tlsCtx, d_connectionStartTime.tv_sec); - if (!tlsSession && d_ds->d_tlsCtx) { - tlsSession = g_sessionCache.getSession(d_ds->getID(), d_connectionStartTime.tv_sec); - } - if (tlsSession) { - handler->setTLSSession(tlsSession); - } - handler->tryConnect(d_ds->tcpFastOpen && isFastOpenEnabled(), d_ds->remote); - d_queries = 0; - - d_handler = std::move(handler); - d_ds->incCurrentConnectionsCount(); - return true; - } - catch (const std::runtime_error& e) { - vinfolog("Connection to downstream server %s failed: %s", d_ds->getName(), e.what()); - d_downstreamFailures++; - if (d_downstreamFailures >= d_ds->retries) { - throw; - } - } + d_pendingQueries.push_back(PendingRequest({sender, std::move(query)})); } - while (d_downstreamFailures < d_ds->retries); - - return false; } void TCPConnectionToBackend::handleTimeout(const struct timeval& now, bool write) @@ -444,31 +512,49 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, F { d_connectionDied = true; - auto& sender = d_sender; - if (!sender->active()) { - // a client timeout occurred, or something like that */ - d_sender.reset(); - return; - } - - if (reason == FailureReason::timeout) { - ++sender->getClientState().tcpDownstreamTimeouts; - } - else if (reason == FailureReason::gaveUp) { - ++sender->getClientState().tcpGaveUp; - } + /* we might be terminated while notifying a query sender */ + d_ds->outstanding -= d_pendingResponses.size(); + auto pendingQueries = std::move(d_pendingQueries); + auto pendingResponses = std::move(d_pendingResponses); + + auto increaseCounters = [reason](std::shared_ptr& sender) { + if (reason == FailureReason::timeout) { + const ClientState* cs = sender->getClientState(); + if (cs) { + ++cs->tcpDownstreamTimeouts; + } + } + else if (reason == FailureReason::gaveUp) { + const ClientState* cs = sender->getClientState(); + if (cs) { + ++cs->tcpGaveUp; + } + } + }; try { if (d_state == State::sendingQueryToBackend) { - sender->notifyIOError(std::move(d_currentQuery.d_idstate), now); + auto sender = d_currentQuery.d_sender; + if (sender->active()) { + increaseCounters(sender); + sender->notifyIOError(std::move(d_currentQuery.d_query.d_idstate), now); + } } - for (auto& query : d_pendingQueries) { - sender->notifyIOError(std::move(query.d_idstate), now); + for (auto& query : pendingQueries) { + auto sender = query.d_sender; + if (sender->active()) { + increaseCounters(sender); + sender->notifyIOError(std::move(query.d_query.d_idstate), now); + } } - for (auto& response : d_pendingResponses) { - sender->notifyIOError(std::move(response.second.d_idstate), now); + for (auto& response : pendingResponses) { + auto sender = response.second.d_sender; + if (sender->active()) { + increaseCounters(sender); + sender->notifyIOError(std::move(response.second.d_query.d_idstate), now); + } } } catch (const std::exception& e) { @@ -507,16 +593,6 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptractive()) { - // a client timeout occurred, or something like that */ - d_connectionDied = true; - - release(); - - return IOState::Done; - } - uint16_t queryId = 0; try { queryId = getQueryIdFromResponse(); @@ -534,33 +610,38 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptrsecond.isXFR()) { + editPayloadID(d_responseBuffer, ntohs(it->second.d_query.d_idstate.origID), 0, false); + + auto sender = it->second.d_sender; + + if (sender->active() && it->second.d_query.isXFR()) { DEBUGLOG("XFR!"); bool done = false; TCPResponse response; response.d_buffer = std::move(d_responseBuffer); response.d_connection = conn; /* we don't move the whole IDS because we will need for the responses to come */ - response.d_idstate.qtype = it->second.d_idstate.qtype; - response.d_idstate.qname = it->second.d_idstate.qname; + response.d_idstate.qtype = it->second.d_query.d_idstate.qtype; + response.d_idstate.qname = it->second.d_query.d_idstate.qname; DEBUGLOG("passing XFRresponse to client connection for "<second.d_xfrStarted = true; - done = isXFRFinished(response, it->second); + it->second.d_query.d_xfrStarted = true; + done = isXFRFinished(response, it->second.d_query); if (done) { d_pendingResponses.erase(it); --conn->d_ds->outstanding; /* marking as idle for now, so we can accept new queries if our queues are empty */ if (d_pendingQueries.empty() && d_pendingResponses.empty()) { + t_downstreamTCPConnectionsManager.moveToIdle(conn); d_state = State::idle; } } sender->handleXFRResponse(now, std::move(response)); if (done) { + t_downstreamTCPConnectionsManager.moveToIdle(conn); d_state = State::idle; - d_sender.reset(); return IOState::Done; } @@ -572,26 +653,24 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptrd_ds->outstanding; - auto ids = std::move(it->second.d_idstate); + auto ids = std::move(it->second.d_query.d_idstate); d_pendingResponses.erase(it); /* marking as idle for now, so we can accept new queries if our queues are empty */ if (d_pendingQueries.empty() && d_pendingResponses.empty()) { + t_downstreamTCPConnectionsManager.moveToIdle(conn); d_state = State::idle; } - DEBUGLOG("passing response to client connection for "<releaseConnection(); - sender->handleResponse(now, TCPResponse(std::move(d_responseBuffer), std::move(ids), conn)); + auto shared = conn; + if (sender->active()) { + DEBUGLOG("passing response to client connection for "<handleResponse(now, TCPResponse(std::move(d_responseBuffer), std::move(ids), conn)); + } if (!d_pendingQueries.empty()) { DEBUGLOG("still have some queries to send"); - d_state = State::sendingQueryToBackend; - d_currentQuery = std::move(d_pendingQueries.front()); - d_currentPos = 0; - d_pendingQueries.pop_front(); - return IOState::NeedWrite; + return queueNextQuery(shared); } else if (!d_pendingResponses.empty()) { DEBUGLOG("still have some responses to read"); @@ -602,24 +681,21 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptrgetNameWithAddr()); } - dnsheader dh; - memcpy(&dh, &d_responseBuffer.at(0), sizeof(dh)); - return ntohs(dh.id); + uint16_t id; + memcpy(&id, &d_responseBuffer.at(0), sizeof(id)); + return ntohs(id); } void TCPConnectionToBackend::setProxyProtocolValuesSent(std::unique_ptr>&& proxyProtocolValuesSent) @@ -668,7 +744,6 @@ bool TCPConnectionToBackend::isXFRFinished(const TCPResponse& response, TCPQuery } auto raw = unknownContent->getRawContent(); auto serial = getSerialFromRawSOAContent(raw); - ++query.d_xfrSerialCount; if (query.d_xfrMasterSerial == 0) { // store the first SOA in our client's connection metadata @@ -696,126 +771,3 @@ bool TCPConnectionToBackend::isXFRFinished(const TCPResponse& response, TCPQuery } return done; } - -std::shared_ptr DownstreamConnectionsManager::getConnectionToDownstream(std::unique_ptr& mplexer, std::shared_ptr& ds, const struct timeval& now) -{ - std::shared_ptr result; - struct timeval freshCutOff = now; - freshCutOff.tv_sec -= 1; - - auto backendId = ds->getID(); - - if (s_cleanupInterval > 0 && (s_nextCleanup == 0 || s_nextCleanup <= now.tv_sec)) { - s_nextCleanup = now.tv_sec + s_cleanupInterval; - cleanupClosedTCPConnections(now); - } - - { - const auto& it = t_downstreamConnections.find(backendId); - if (it != t_downstreamConnections.end()) { - auto& list = it->second; - while (!list.empty()) { - result = std::move(list.back()); - list.pop_back(); - - result->setReused(); - /* for connections that have not been used very recently, - check whether they have been closed in the meantime */ - if (freshCutOff < result->getLastDataReceivedTime()) { - /* used recently enough, skip the check */ - ++ds->tcpReusedConnections; - return result; - } - - if (isTCPSocketUsable(result->getHandle())) { - ++ds->tcpReusedConnections; - return result; - } - - /* otherwise let's try the next one, if any */ - } - } - } - - return std::make_shared(ds, mplexer, now); -} - -void DownstreamConnectionsManager::releaseDownstreamConnection(std::shared_ptr&& conn) -{ - if (conn == nullptr) { - return; - } - - if (!conn->canBeReused()) { - conn.reset(); - return; - } - - const auto& ds = conn->getDS(); - { - auto& list = t_downstreamConnections[ds->getID()]; - while (list.size() >= s_maxCachedConnectionsPerDownstream) { - /* too many connections queued already */ - list.pop_front(); - } - - list.push_back(std::move(conn)); - } -} - -void DownstreamConnectionsManager::cleanupClosedTCPConnections(struct timeval now) -{ - struct timeval freshCutOff = now; - freshCutOff.tv_sec -= 1; - - for (auto dsIt = t_downstreamConnections.begin(); dsIt != t_downstreamConnections.end(); ) { - for (auto connIt = dsIt->second.begin(); connIt != dsIt->second.end(); ) { - if (!(*connIt)) { - ++connIt; - continue; - } - - /* don't bother checking freshly used connections */ - if (freshCutOff < (*connIt)->getLastDataReceivedTime()) { - ++connIt; - continue; - } - - if (isTCPSocketUsable((*connIt)->getHandle())) { - ++connIt; - } - else { - connIt = dsIt->second.erase(connIt); - } - } - - if (!dsIt->second.empty()) { - ++dsIt; - } - else { - dsIt = t_downstreamConnections.erase(dsIt); - } - } -} - -size_t DownstreamConnectionsManager::clear() -{ - size_t count = 0; - for (const auto& downstream : t_downstreamConnections) { - count += downstream.second.size(); - } - - t_downstreamConnections.clear(); - - return count; -} - -void setMaxCachedTCPConnectionsPerDownstream(size_t max) -{ - DownstreamConnectionsManager::setMaxCachedConnectionsPerDownstream(max); -} - -thread_local map>> DownstreamConnectionsManager::t_downstreamConnections; -size_t DownstreamConnectionsManager::s_maxCachedConnectionsPerDownstream{10}; -time_t DownstreamConnectionsManager::s_nextCleanup{0}; -uint16_t DownstreamConnectionsManager::s_cleanupInterval{60}; diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh index 3745de96571d..f85fc36e5e72 100644 --- a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh +++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh @@ -7,15 +7,15 @@ #include "dnsdist.hh" #include "dnsdist-tcp.hh" -class TCPConnectionToBackend : public std::enable_shared_from_this +class ConnectionToBackend : public std::enable_shared_from_this { public: - TCPConnectionToBackend(std::shared_ptr& ds, std::unique_ptr& mplexer, const struct timeval& now): d_responseBuffer(s_maxPacketCacheEntrySize), d_mplexer(mplexer), d_ds(ds), d_connectionStartTime(now), d_lastDataReceivedTime(now), d_enableFastOpen(ds->tcpFastOpen) + ConnectionToBackend(const std::shared_ptr& ds, std::unique_ptr& mplexer, const struct timeval& now): d_connectionStartTime(now), d_lastDataReceivedTime(now), d_ds(ds), d_mplexer(mplexer), d_enableFastOpen(ds->tcpFastOpen) { reconnect(); } - ~TCPConnectionToBackend(); + virtual ~ConnectionToBackend(); int getHandle() const { @@ -26,6 +26,16 @@ public: return d_handler->getDescriptor(); } + /* whether the underlying socket has been closed under our feet, basically */ + bool isUsable() const + { + if (!d_handler) { + return false; + } + + return d_handler->isUsable(); + } + const std::shared_ptr& getDS() const { return d_ds; @@ -46,11 +56,6 @@ public: return d_fresh; } - void incQueries() - { - ++d_queries; - } - void setReused() { d_fresh = false; @@ -66,43 +71,54 @@ public: return d_enableFastOpen; } - /* whether we can accept new queries FOR THE SAME CLIENT */ - bool canAcceptNewQueries() const + /* whether a connection can be used now */ + bool canBeReused(bool sameClient = false) const { if (d_connectionDied) { return false; } - if ((d_pendingQueries.size() + d_pendingResponses.size()) >= d_ds->d_maxInFlightQueriesPerConn) { + /* we can't reuse a connection where a proxy protocol payload has been sent, + since: + - it cannot be reused for a different client + - we might have different TLV values for each query + */ + if (d_ds && d_ds->useProxyProtocol == true && !sameClient) { + return false; + } + + if (reachedMaxStreamID()) { return false; } - return true; - } + if (reachedMaxConcurrentQueries()) { + return false; + } - bool isIdle() const - { - return d_state == State::idle && d_pendingQueries.size() == 0 && d_pendingResponses.size() == 0; + return true; } - /* whether a connection can be reused for a different client */ - bool canBeReused() const + /* full now but will become usable later */ + bool willBeReusable(bool sameClient) const { - if (d_connectionDied) { + if (d_connectionDied || reachedMaxStreamID()) { return false; } - /* we can't reuse a connection where a proxy protocol payload has been sent, - since: - - it cannot be reused for a different client - - we might have different TLV values for each query - */ + if (d_ds && d_ds->useProxyProtocol == true) { - return false; + return sameClient; } + return true; } - bool matchesTLVs(const std::unique_ptr>& tlvs) const; + virtual bool reachedMaxStreamID() const = 0; + virtual bool reachedMaxConcurrentQueries() const = 0; + virtual bool isIdle() const = 0; + virtual void release() = 0; + virtual void stopIO() + { + } bool matches(const std::shared_ptr& ds) const { @@ -112,43 +128,32 @@ public: return ds == d_ds; } - void queueQuery(std::shared_ptr& sender, TCPQuery&& query); - void handleTimeout(const struct timeval& now, bool write); - void release(); - - void setProxyProtocolValuesSent(std::unique_ptr>&& proxyProtocolValuesSent); + virtual void queueQuery(std::shared_ptr& sender, TCPQuery&& query) = 0; + virtual void handleTimeout(const struct timeval& now, bool write) = 0; struct timeval getLastDataReceivedTime() const { return d_lastDataReceivedTime; } - std::string toString() const - { - ostringstream o; - o << "TCP connection to backend "<<(d_ds ? d_ds->getName() : "empty")<<" over FD "<<(d_handler ? std::to_string(d_handler->getDescriptor()) : "no socket")<<", state is "<<(int)d_state<<", io state is "<<(d_ioState ? std::to_string((int)d_ioState->getState()) : "empty")<<", queries count is "<& conn, const struct timeval& now); - static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param); - static IOState queueNextQuery(std::shared_ptr& conn); - static IOState sendQuery(std::shared_ptr& conn, const struct timeval& now); - static bool isXFRFinished(const TCPResponse& response, TCPQuery& query); - - IOState handleResponse(std::shared_ptr& conn, const struct timeval& now); - uint16_t getQueryIdFromResponse(); +protected: bool reconnect(); - void notifyAllQueriesFailed(const struct timeval& now, FailureReason reason); - bool needProxyProtocolPayload() const + + boost::optional getBackendHealthCheckTTD(const struct timeval& now) const { - return !d_proxyProtocolPayloadSent && (d_ds && d_ds->useProxyProtocol); + if (d_ds == nullptr) { + throw std::runtime_error("getBackendReadTTD() without any backend selected"); + } + if (d_ds->checkTimeout == 0) { + return boost::none; + } + + struct timeval res = now; + res.tv_sec += d_ds->checkTimeout; + + return res; } boost::optional getBackendReadTTD(const struct timeval& now) const @@ -196,40 +201,128 @@ private: return res; } - PacketBuffer d_responseBuffer; - std::deque d_pendingQueries; - std::unordered_map d_pendingResponses; + struct timeval d_connectionStartTime; + struct timeval d_lastDataReceivedTime; + const std::shared_ptr d_ds{nullptr}; + std::shared_ptr d_sender{nullptr}; std::unique_ptr& d_mplexer; - std::unique_ptr> d_proxyProtocolValuesSent{nullptr}; std::unique_ptr d_handler{nullptr}; std::unique_ptr d_ioState{nullptr}; - std::shared_ptr d_ds{nullptr}; - std::shared_ptr d_sender{nullptr}; - TCPQuery d_currentQuery; - struct timeval d_connectionStartTime; - struct timeval d_lastDataReceivedTime; - size_t d_currentPos{0}; uint64_t d_queries{0}; - uint64_t d_downstreamFailures{0}; - uint16_t d_responseSize{0}; - State d_state{State::idle}; - bool d_fresh{true}; + uint32_t d_highestStreamID{0}; + uint16_t d_downstreamFailures{0}; + bool d_proxyProtocolPayloadSent{false}; bool d_enableFastOpen{false}; bool d_connectionDied{false}; - bool d_proxyProtocolPayloadSent{false}; + bool d_fresh{true}; }; -class DownstreamConnectionsManager +class TCPConnectionToBackend : public ConnectionToBackend { public: - static std::shared_ptr getConnectionToDownstream(std::unique_ptr& mplexer, std::shared_ptr& ds, const struct timeval& now); - static void releaseDownstreamConnection(std::shared_ptr&& conn); - static void cleanupClosedTCPConnections(struct timeval now); - static size_t clear(); + TCPConnectionToBackend(const std::shared_ptr& ds, std::unique_ptr& mplexer, const struct timeval& now, std::string&& /* proxyProtocolPayload*, unused but there to match the HTTP2 connections, so we can use the same templated connections manager class */): ConnectionToBackend(ds, mplexer, now), d_responseBuffer(s_maxPacketCacheEntrySize) + { + } + + virtual ~TCPConnectionToBackend(); + + bool isIdle() const override + { + return d_state == State::idle && d_pendingQueries.size() == 0 && d_pendingResponses.size() == 0; + } - static void setMaxCachedConnectionsPerDownstream(size_t max) + bool reachedMaxStreamID() const override { - s_maxCachedConnectionsPerDownstream = max; + /* TCP/DoT has only 2^16 usable identifiers, DoH has 2^32 */ + const uint32_t maximumStreamID = std::numeric_limits::max() - 1; + return d_highestStreamID == maximumStreamID; + } + + bool reachedMaxConcurrentQueries() const override + { + const size_t concurrent = d_pendingQueries.size() + d_pendingResponses.size(); + if (concurrent > 0 && concurrent >= d_ds->d_maxInFlightQueriesPerConn) { + return true; + } + return false; + } + bool matchesTLVs(const std::unique_ptr>& tlvs) const; + + void queueQuery(std::shared_ptr& sender, TCPQuery&& query) override; + void handleTimeout(const struct timeval& now, bool write) override; + void release() override; + + std::string toString() const override + { + ostringstream o; + o << "TCP connection to backend "<<(d_ds ? d_ds->getName() : "empty")<<" over FD "<<(d_handler ? std::to_string(d_handler->getDescriptor()) : "no socket")<<", state is "<<(int)d_state<<", io state is "<<(d_ioState ? d_ioState->getState() : "empty")<<", queries count is "<>&& proxyProtocolValuesSent); + +private: + /* waitingForResponseFromBackend is a state where we have not yet started reading the size, + so we can still switch to sending instead */ + enum class State : uint8_t { idle, sendingQueryToBackend, waitingForResponseFromBackend, readingResponseSizeFromBackend, readingResponseFromBackend }; + enum class FailureReason : uint8_t { /* too many attempts */ gaveUp, timeout, unexpectedQueryID }; + + static void handleIO(std::shared_ptr& conn, const struct timeval& now); + static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param); + static IOState queueNextQuery(std::shared_ptr& conn); + static IOState sendQuery(std::shared_ptr& conn, const struct timeval& now); + static bool isXFRFinished(const TCPResponse& response, TCPQuery& query); + + IOState handleResponse(std::shared_ptr& conn, const struct timeval& now); + uint16_t getQueryIdFromResponse() const; + void notifyAllQueriesFailed(const struct timeval& now, FailureReason reason); + bool needProxyProtocolPayload() const + { + return !d_proxyProtocolPayloadSent && (d_ds && d_ds->useProxyProtocol); + } + + class PendingRequest + { + public: + std::shared_ptr d_sender{nullptr}; + TCPQuery d_query; + }; + + PacketBuffer d_responseBuffer; + std::deque d_pendingQueries; + std::unordered_map d_pendingResponses; + std::unique_ptr> d_proxyProtocolValuesSent{nullptr}; + PendingRequest d_currentQuery; + size_t d_currentPos{0}; + uint16_t d_responseSize{0}; + State d_state{State::idle}; +}; + +template class DownstreamConnectionsManager +{ + struct SequencedTag {}; + struct OrderedTag {}; + + typedef multi_index_container< + std::shared_ptr, + indexed_by < + ordered_unique, + identity> + >, + /* new elements are added to the front of the sequence */ + sequenced > + > + > list_t; + struct ConnectionLists + { + list_t d_actives; + list_t d_idles; + }; + +public: + static void setMaxIdleConnectionsPerDownstream(size_t max) + { + s_maxIdleConnectionsPerDownstream = max; } static void setCleanupInterval(uint16_t interval) @@ -237,9 +330,260 @@ public: s_cleanupInterval = interval; } -private: - static thread_local map>> t_downstreamConnections; - static size_t s_maxCachedConnectionsPerDownstream; - static time_t s_nextCleanup; + static void setMaxIdleTime(uint16_t max) + { + s_maxIdleTime = max; + } + + std::shared_ptr getConnectionToDownstream(std::unique_ptr& mplexer, const std::shared_ptr& ds, const struct timeval& now, std::string&& proxyProtocolPayload) + { + struct timeval freshCutOff = now; + freshCutOff.tv_sec -= 1; + + auto backendId = ds->getID(); + + cleanupClosedConnections(now); + + const bool haveProxyProtocol = ds->useProxyProtocol || !proxyProtocolPayload.empty(); + if (!haveProxyProtocol) { + const auto& it = d_downstreamConnections.find(backendId); + if (it != d_downstreamConnections.end()) { + /* first scan idle connections, more recent first */ + auto entry = findUsableConnectionInList(now, freshCutOff, it->second.d_idles, true); + if (entry) { + ++ds->tcpReusedConnections; + it->second.d_actives.insert(entry); + return entry; + } + + /* then scan actives ones, more recent first as well */ + entry = findUsableConnectionInList(now, freshCutOff, it->second.d_actives, false); + if (entry) { + ++ds->tcpReusedConnections; + return entry; + } + } + } + + auto newConnection = std::make_shared(ds, mplexer, now, std::move(proxyProtocolPayload)); + if (!haveProxyProtocol) { + auto& list = d_downstreamConnections[backendId].d_actives; + list.template get().push_front(newConnection); + } + + return newConnection; + } + + void cleanupClosedConnections(const struct timeval& now) + { + if (s_cleanupInterval == 0 || (d_nextCleanup != 0 && d_nextCleanup > now.tv_sec)) { + return; + } + + d_nextCleanup = now.tv_sec + s_cleanupInterval; + + struct timeval freshCutOff = now; + freshCutOff.tv_sec -= 1; + struct timeval idleCutOff = now; + idleCutOff.tv_sec -= s_maxIdleTime; + + for (auto dsIt = d_downstreamConnections.begin(); dsIt != d_downstreamConnections.end(); ) { + cleanUpList(dsIt->second.d_idles, now, freshCutOff, idleCutOff); + cleanUpList(dsIt->second.d_actives, now, freshCutOff, idleCutOff); + + if (dsIt->second.d_idles.empty() && dsIt->second.d_actives.empty()) { + dsIt = d_downstreamConnections.erase(dsIt); + } + else { + ++dsIt; + } + } + } + + size_t clear() + { + size_t count = 0; + for (const auto& downstream : d_downstreamConnections) { + count += downstream.second.d_actives.size(); + for (auto& conn : downstream.second.d_actives) { + conn->stopIO(); + } + count += downstream.second.d_idles.size(); + for (auto& conn : downstream.second.d_idles) { + conn->stopIO(); + } + } + + d_downstreamConnections.clear(); + return count; + } + + size_t count() const + { + return getActiveCount() + getIdleCount(); + } + + size_t getActiveCount() const + { + size_t count = 0; + for (const auto& downstream : d_downstreamConnections) { + count += downstream.second.d_actives.size(); + } + return count; + } + + size_t getIdleCount() const + { + size_t count = 0; + for (const auto& downstream : d_downstreamConnections) { + count += downstream.second.d_idles.size(); + } + return count; + } + + bool removeDownstreamConnection(std::shared_ptr& conn) + { + auto backendIt = d_downstreamConnections.find(conn->getDS()->getID()); + if (backendIt == d_downstreamConnections.end()) { + return false; + } + + /* idle list first */ + { + auto it = backendIt->second.d_idles.find(conn); + if (it != backendIt->second.d_idles.end()) { + backendIt->second.d_idles.erase(it); + return true; + } + } + /* then active */ + { + auto it = backendIt->second.d_actives.find(conn); + if (it != backendIt->second.d_actives.end()) { + backendIt->second.d_actives.erase(it); + return true; + } + } + + return false; + } + + bool moveToIdle(std::shared_ptr& conn) + { + auto backendIt = d_downstreamConnections.find(conn->getDS()->getID()); + if (backendIt == d_downstreamConnections.end()) { + return false; + } + + auto it = backendIt->second.d_actives.find(conn); + if (it == backendIt->second.d_actives.end()) { + return false; + } + + backendIt->second.d_actives.erase(it); + + if (backendIt->second.d_idles.size() >= s_maxIdleConnectionsPerDownstream) { + backendIt->second.d_idles.template get().pop_back(); + } + + backendIt->second.d_idles.template get().push_front(conn); + return true; + } + +protected: + + void cleanUpList(list_t& list, const struct timeval& now, const struct timeval& freshCutOff, const struct timeval& idleCutOff) + { + auto& sidx = list.template get(); + for (auto connIt = sidx.begin(); connIt != sidx.end(); ) { + if (!(*connIt)) { + connIt = sidx.erase(connIt); + continue; + } + + auto& entry = *connIt; + + /* don't bother checking freshly used connections */ + if (freshCutOff < entry->getLastDataReceivedTime()) { + ++connIt; + continue; + } + + if (entry->isIdle() && entry->getLastDataReceivedTime() < idleCutOff) { + /* idle for too long */ + connIt = sidx.erase(connIt); + continue; + } + + if (entry->isUsable()) { + ++connIt; + continue; + } + + connIt = sidx.erase(connIt); + } + } + + std::shared_ptr findUsableConnectionInList(const struct timeval& now, const struct timeval& freshCutOff, list_t& list, bool removeIfFound) + { + auto& sidx = list.template get(); + for (auto listIt = sidx.begin(); listIt != sidx.end(); ) { + if (!(*listIt)) { + listIt = sidx.erase(listIt); + continue; + } + + auto& entry = *listIt; + if (isConnectionUsable(entry, now, freshCutOff)) { + entry->setReused(); + // make a copy since the iterator will be invalidated after erasing + auto result = entry; + if (removeIfFound) { + sidx.erase(listIt); + } + return result; + } + + if (entry->willBeReusable(false)) { + ++listIt; + continue; + } + + /* that connection will not be usable later, no need to keep it in that list */ + listIt = sidx.erase(listIt); + } + + return nullptr; + } + + bool isConnectionUsable(const std::shared_ptr& conn, const struct timeval& now, const struct timeval& freshCutOff) + { + if (!conn->canBeReused()) { + return false; + } + + /* for connections that have not been used very recently, + check whether they have been closed in the meantime */ + if (freshCutOff < conn->getLastDataReceivedTime()) { + /* used recently enough, skip the check */ + return true; + } + + return conn->isUsable(); + } + + static size_t s_maxIdleConnectionsPerDownstream; static uint16_t s_cleanupInterval; + static uint16_t s_maxIdleTime; + + std::map d_downstreamConnections; + + time_t d_nextCleanup{0}; }; + +template size_t DownstreamConnectionsManager::s_maxIdleConnectionsPerDownstream{10}; +template uint16_t DownstreamConnectionsManager::s_cleanupInterval{60}; +template uint16_t DownstreamConnectionsManager::s_maxIdleTime{300}; + +using DownstreamTCPConnectionsManager = DownstreamConnectionsManager; +extern thread_local DownstreamTCPConnectionsManager t_downstreamTCPConnectionsManager; diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh index 9d392006c958..9ed8b6b3fa4d 100644 --- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh +++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh @@ -13,12 +13,13 @@ public: LocalHolders holders; LocalStateHolder > localRespRuleActions; std::unique_ptr mplexer{nullptr}; + int crossProtocolResponsesPipe{-1}; }; class IncomingTCPConnectionState : public TCPQuerySender, public std::enable_shared_from_this { public: - IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, timeval{g_tcpRecvTimeout,0}, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(make_unique(*threadData.mplexer, d_ci.fd)), d_connectionStartTime(now) + IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_ci(std::move(ci)), d_handler(d_ci.fd, timeval{g_tcpRecvTimeout,0}, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_connectionStartTime(now), d_ioState(make_unique(*threadData.mplexer, d_ci.fd)), d_threadData(threadData) { d_origDest.reset(); d_origDest.sin4.sin_family = d_ci.remote.sin4.sin_family; @@ -104,9 +105,9 @@ public: return false; } - std::shared_ptr getActiveDownstreamConnection(const std::shared_ptr& ds, const std::unique_ptr>& tlvs); + std::shared_ptr getOwnedDownstreamConnection(const std::shared_ptr& ds, const std::unique_ptr>& tlvs); std::shared_ptr getDownstreamConnection(std::shared_ptr& ds, const std::unique_ptr>& tlvs, const struct timeval& now); - void registerActiveDownstreamConnection(std::shared_ptr& conn); + void registerOwnedDownstreamConnection(std::shared_ptr& conn); static size_t clearAllDownstreamConnections(); @@ -132,37 +133,37 @@ static void handleTimeout(std::shared_ptr& state, bo return d_ioState != nullptr; } - const ClientState& getClientState() override + const ClientState* getClientState() const override { - return *d_ci.cs; + return d_ci.cs; } std::string toString() const { ostringstream o; - o << "Incoming TCP connection from "<getState()) : "empty")<<", queries count is "<getState() : "empty")<<", queries count is "<, std::deque>> d_activeConnectionsToBackend; - PacketBuffer d_buffer; - std::deque d_queuedResponses; - TCPClientThreadData& d_threadData; TCPResponse d_currentResponse; + std::map, std::deque>> d_ownedConnectionsToBackend; + std::deque d_queuedResponses; + PacketBuffer d_buffer; ConnectionInfo d_ci; ComboAddress d_origDest; ComboAddress d_proxiedRemote; ComboAddress d_proxiedDestination; TCPIOHandler d_handler; - std::unique_ptr d_ioState{nullptr}; - std::unique_ptr> d_proxyProtocolValues{nullptr}; struct timeval d_connectionStartTime; struct timeval d_handshakeDoneTime; struct timeval d_firstQuerySizeReadTime; struct timeval d_querySizeReadTime; struct timeval d_queryReadTime; + std::unique_ptr d_ioState{nullptr}; + std::unique_ptr> d_proxyProtocolValues{nullptr}; + TCPClientThreadData& d_threadData; size_t d_currentPos{0}; size_t d_proxyProtocolNeed{0}; size_t d_queriesCount{0}; diff --git a/pdns/dnsdistdist/dnsdist-tcp.hh b/pdns/dnsdistdist/dnsdist-tcp.hh index 99da15c09435..9154f2f650f2 100644 --- a/pdns/dnsdistdist/dnsdist-tcp.hh +++ b/pdns/dnsdistdist/dnsdist-tcp.hh @@ -21,12 +21,20 @@ */ #pragma once +#include +#include "iputils.hh" +#include "dnsdist.hh" + struct ConnectionInfo { ConnectionInfo(ClientState* cs_) : cs(cs_), fd(-1) { } + ConnectionInfo(ClientState* cs_, const ComboAddress remote_) : + remote(remote_), cs(cs_), fd(-1) + { + } ConnectionInfo(ConnectionInfo&& rhs) : remote(rhs.remote), cs(rhs.cs), fd(rhs.fd) { @@ -76,7 +84,7 @@ struct InternalQuery } InternalQuery(InternalQuery&& rhs) : - d_idstate(std::move(rhs.d_idstate)), d_buffer(std::move(rhs.d_buffer)), d_proxyProtocolPayload(std::move(rhs.d_proxyProtocolPayload)), d_xfrMasterSerial(rhs.d_xfrMasterSerial), d_xfrSerialCount(rhs.d_xfrSerialCount), d_xfrMasterSerialCount(rhs.d_xfrMasterSerialCount), d_proxyProtocolPayloadAdded(rhs.d_proxyProtocolPayloadAdded) + d_idstate(std::move(rhs.d_idstate)), d_proxyProtocolPayload(std::move(rhs.d_proxyProtocolPayload)), d_buffer(std::move(rhs.d_buffer)), d_xfrMasterSerial(rhs.d_xfrMasterSerial), d_xfrSerialCount(rhs.d_xfrSerialCount), d_downstreamFailures(rhs.d_downstreamFailures), d_xfrMasterSerialCount(rhs.d_xfrMasterSerialCount), d_proxyProtocolPayloadAdded(rhs.d_proxyProtocolPayloadAdded) { } InternalQuery& operator=(InternalQuery&& rhs) @@ -86,6 +94,7 @@ struct InternalQuery d_proxyProtocolPayload = std::move(rhs.d_proxyProtocolPayload); d_xfrMasterSerial = rhs.d_xfrMasterSerial; d_xfrSerialCount = rhs.d_xfrSerialCount; + d_downstreamFailures = rhs.d_downstreamFailures; d_xfrMasterSerialCount = rhs.d_xfrMasterSerialCount; d_proxyProtocolPayloadAdded = rhs.d_proxyProtocolPayloadAdded; return *this; @@ -100,10 +109,11 @@ struct InternalQuery } IDState d_idstate; - PacketBuffer d_buffer; std::string d_proxyProtocolPayload; + PacketBuffer d_buffer; uint32_t d_xfrMasterSerial{0}; uint32_t d_xfrSerialCount{0}; + uint32_t d_downstreamFailures{0}; uint8_t d_xfrMasterSerialCount{0}; bool d_xfrStarted{false}; bool d_proxyProtocolPayloadAdded{false}; @@ -111,7 +121,7 @@ struct InternalQuery using TCPQuery = InternalQuery; -class TCPConnectionToBackend; +class ConnectionToBackend; struct TCPResponse : public TCPQuery { @@ -121,13 +131,13 @@ struct TCPResponse : public TCPQuery memset(&d_cleartextDH, 0, sizeof(d_cleartextDH)); } - TCPResponse(PacketBuffer&& buffer, IDState&& state, std::shared_ptr conn) : + TCPResponse(PacketBuffer&& buffer, IDState&& state, std::shared_ptr conn) : TCPQuery(std::move(buffer), std::move(state)), d_connection(conn) { memset(&d_cleartextDH, 0, sizeof(d_cleartextDH)); } - std::shared_ptr d_connection{nullptr}; + std::shared_ptr d_connection{nullptr}; dnsheader d_cleartextDH; bool d_selfGenerated{false}; }; @@ -140,7 +150,7 @@ public: } virtual bool active() const = 0; - virtual const ClientState& getClientState() = 0; + virtual const ClientState* getClientState() const = 0; virtual void handleResponse(const struct timeval& now, TCPResponse&& response) = 0; virtual void handleXFRResponse(const struct timeval& now, TCPResponse&& response) = 0; virtual void notifyIOError(IDState&& query, const struct timeval& now) = 0; @@ -188,7 +198,7 @@ public: uint64_t pos = d_pos++; ++d_queued; - return d_tcpclientthreads.at(pos % d_numthreads).d_newConnectionPipe; + return d_tcpclientthreads.at(pos % d_numthreads).d_newConnectionPipe.getHandle(); } bool passConnectionToThread(std::unique_ptr&& conn) @@ -198,10 +208,11 @@ public: } uint64_t pos = d_pos++; - auto pipe = d_tcpclientthreads.at(pos % d_numthreads).d_newConnectionPipe; + auto pipe = d_tcpclientthreads.at(pos % d_numthreads).d_newConnectionPipe.getHandle(); auto tmp = conn.release(); if (write(pipe, &tmp, sizeof(tmp)) != sizeof(tmp)) { + ++g_stats.tcpQueryPipeFull; delete tmp; tmp = nullptr; return false; @@ -217,10 +228,11 @@ public: } uint64_t pos = d_pos++; - auto pipe = d_tcpclientthreads.at(pos % d_numthreads).d_crossProtocolQueryPipe; + auto pipe = d_tcpclientthreads.at(pos % d_numthreads).d_crossProtocolQueriesPipe.getHandle(); auto tmp = cpq.release(); if (write(pipe, &tmp, sizeof(tmp)) != sizeof(tmp)) { + ++g_stats.tcpCrossProtocolQueryPipeFull; delete tmp; tmp = nullptr; return false; @@ -249,62 +261,30 @@ public: --d_queued; } +private: void addTCPClientThread(); -private: struct TCPWorkerThread { TCPWorkerThread() { } - TCPWorkerThread(int newConnPipe, int crossProtocolPipe) : - d_newConnectionPipe(newConnPipe), d_crossProtocolQueryPipe(crossProtocolPipe) - { - } - - TCPWorkerThread(TCPWorkerThread&& rhs) : - d_newConnectionPipe(rhs.d_newConnectionPipe), d_crossProtocolQueryPipe(rhs.d_crossProtocolQueryPipe) - { - rhs.d_newConnectionPipe = -1; - rhs.d_crossProtocolQueryPipe = -1; - } - - TCPWorkerThread& operator=(TCPWorkerThread&& rhs) + TCPWorkerThread(int newConnPipe, int crossProtocolQueriesPipe, int crossProtocolResponsesPipe) : + d_newConnectionPipe(newConnPipe), d_crossProtocolQueriesPipe(crossProtocolQueriesPipe), d_crossProtocolResponsesPipe(crossProtocolResponsesPipe) { - if (d_newConnectionPipe != -1) { - close(d_newConnectionPipe); - } - if (d_crossProtocolQueryPipe != -1) { - close(d_crossProtocolQueryPipe); - } - - d_newConnectionPipe = rhs.d_newConnectionPipe; - d_crossProtocolQueryPipe = rhs.d_crossProtocolQueryPipe; - rhs.d_newConnectionPipe = -1; - rhs.d_crossProtocolQueryPipe = -1; - - return *this; } + TCPWorkerThread(TCPWorkerThread&& rhs) = default; + TCPWorkerThread& operator=(TCPWorkerThread&& rhs) = default; TCPWorkerThread(const TCPWorkerThread& rhs) = delete; TCPWorkerThread& operator=(const TCPWorkerThread&) = delete; - ~TCPWorkerThread() - { - if (d_newConnectionPipe != -1) { - close(d_newConnectionPipe); - } - if (d_crossProtocolQueryPipe != -1) { - close(d_crossProtocolQueryPipe); - } - } - - int d_newConnectionPipe{-1}; - int d_crossProtocolQueryPipe{-1}; + FDWrapper d_newConnectionPipe; + FDWrapper d_crossProtocolQueriesPipe; + FDWrapper d_crossProtocolResponsesPipe; }; - std::mutex d_mutex; std::vector d_tcpclientthreads; stat_t d_numthreads{0}; stat_t d_pos{0}; diff --git a/pdns/dnsdistdist/dnsdist-tsan.supp b/pdns/dnsdistdist/dnsdist-tsan.supp index 39ffe6cd485e..c8c2d50e340d 100644 --- a/pdns/dnsdistdist/dnsdist-tsan.supp +++ b/pdns/dnsdistdist/dnsdist-tsan.supp @@ -11,3 +11,4 @@ race:DownstreamState::setDown race:DownstreamState::setUp race:DownstreamState::setAuto race:updateHealthCheckResult +race:carbonDumpThread diff --git a/pdns/dnsdistdist/dnsdist-web.hh b/pdns/dnsdistdist/dnsdist-web.hh index ea869274c475..fea62bedffe5 100644 --- a/pdns/dnsdistdist/dnsdist-web.hh +++ b/pdns/dnsdistdist/dnsdist-web.hh @@ -1,7 +1,9 @@ #pragma once -void setWebserverAPIKey(const boost::optional apiKey); -void setWebserverPassword(const std::string& password); +#include "credentials.hh" + +void setWebserverAPIKey(std::unique_ptr&& apiKey); +void setWebserverPassword(std::unique_ptr&& password); void setWebserverACL(const std::string& acl); void setWebserverCustomHeaders(const boost::optional > customHeaders); void setWebserverStatsRequireAuthentication(bool); @@ -10,3 +12,4 @@ void setWebserverMaxConcurrentConnections(size_t); void dnsdistWebserverThread(int sock, const ComboAddress& local); void registerBuiltInWebHandlers(); +void clearWebHandlers(); diff --git a/pdns/dnsdistdist/dnsdist.service.in b/pdns/dnsdistdist/dnsdist.service.in index 3e521f22a8a3..bb11a26dd7e3 100644 --- a/pdns/dnsdistdist/dnsdist.service.in +++ b/pdns/dnsdistdist/dnsdist.service.in @@ -11,6 +11,7 @@ ExecStartPre=@bindir@/dnsdist --check-config ExecStart=@bindir@/dnsdist --supervised --disable-syslog User=@service_user@ Group=@service_group@ +SyslogIdentifier=dnsdist Type=notify Restart=on-failure RestartSec=2 diff --git a/pdns/dnsdistdist/docs/advanced/passing-source-address.rst b/pdns/dnsdistdist/docs/advanced/passing-source-address.rst index ea006fb58d3b..54f54c65d4c2 100644 --- a/pdns/dnsdistdist/docs/advanced/passing-source-address.rst +++ b/pdns/dnsdistdist/docs/advanced/passing-source-address.rst @@ -77,6 +77,8 @@ Please also note that the maximum size of a Proxy Protocol header dnsdist is wil dnsdist 1.5.0 only supports outgoing Proxy Protocol. Support for parsing incoming Proxy Protocol headers has been implemented in 1.6.0, except for DoH where it does not make sense anyway, since HTTP headers already provide a mechanism for that. +Both the PowerDNS Authoritative Server and the Recursor can parse PROXYv2 headers, if configured to do so with their `proxy-protocol-from` setting. + Influence on caching -------------------- diff --git a/pdns/dnsdistdist/docs/advanced/tuning.rst b/pdns/dnsdistdist/docs/advanced/tuning.rst index 9afa06c7d0cb..6d85961bf4a9 100644 --- a/pdns/dnsdistdist/docs/advanced/tuning.rst +++ b/pdns/dnsdistdist/docs/advanced/tuning.rst @@ -13,8 +13,8 @@ First, a few words about :program:`dnsdist` architecture: * One or more webserver threads handle queries to the internal webserver, plus one thread per HTTP connection * A SNMP thread handles SNMP operations, when enabled. -UDP and DNS over HTTPS ------------------------ +UDP and incoming DNS over HTTPS +------------------------------- .. figure:: ../imgs/DNSDistUDP.png :align: center @@ -70,6 +70,16 @@ For DNS over HTTPS, every :func:`addDOHLocal` directive adds a new thread dealin When dealing with a large traffic load, it might happen that the internal pipe used to pass queries between the threads handling the incoming connections and the one getting a response from the backend become full too quickly, degrading performance and causing timeouts. This can be prevented by increasing the size of the internal pipe buffer, via the `internalPipeBufferSize` option of :func:`addDOHLocal`. Setting a value of `1048576` is known to yield good results on Linux. +Outgoing DoH +------------ + +Starting with 1.7.0, dnsdist supports communicating with the backend using DNS over HTTPS. The incoming queries, after the processing of rules if any, are passed to one of the DoH workers over a pipe. The DoH worker handles the communication with the backend, retrieves the response, and either responds directly to the client (queries coming over UDP) or pass it back over a pipe to the initial thread (queries coming over TCP, DoT or DoH). +The number of outgoing DoH worker threads can be configured using :func:`setOutgoingDoHWorkerThreads`. + +.. figure:: ../imgs/DNSDistOutgoingDoH.png + :align: center + :alt: DNSDist outgoing DoH design + TCP and DNS over TLS -------------------- diff --git a/pdns/dnsdistdist/docs/changelog.rst b/pdns/dnsdistdist/docs/changelog.rst index a3137d4b800c..4a51966cab76 100644 --- a/pdns/dnsdistdist/docs/changelog.rst +++ b/pdns/dnsdistdist/docs/changelog.rst @@ -1,6 +1,446 @@ Changelog ========= +.. changelog:: + :version: 1.7.0-beta1 + :released: 16th of November 2021 + + .. change:: + :tags: Improvements + :pullreq: 10646 + + Convert make_pair to emplace (Rosen Penev) + + .. change:: + :tags: Improvements + :pullreq: 10795 + :tickets: 10651 + + Add syslog identifier to service file + + .. change:: + :tags: New Features + :pullreq: 10815 + :tickets: 4993 + + Add range support for dynamic blocks + + .. change:: + :tags: Bug Fixes, DNS over HTTPS + :pullreq: 10845 + + Keep watching idle DoH backend connections + + .. change:: + :tags: Improvements, Performance + :pullreq: 10862 + + Use the same outgoing TCP connection for different clients + + .. change:: + :tags: Improvements + :pullreq: 10868 + + Get rid of make_pair (Rosen Penev) + + .. change:: + :tags: Improvements + :pullreq: 10870 + + Use make_unique instead of new (Rosen Penev) + + .. change:: + :tags: Bug Fixes + :pullreq: 10874 + + Properly handle I/O exceptions in the health checker + + .. change:: + :tags: Improvements, DNS over HTTPS, Performance + :pullreq: 10875 + + Read as many DoH responses as possible before yielding + + .. change:: + :tags: Improvements, DNS over HTTPS, Performance + :pullreq: 10876 + + Stop over-allocating for DoH queries + + .. change:: + :tags: Improvements, Protobuf, DNSTAP + :pullreq: 10879 + :tickets: 9103 + + Support DoT, DoH and DNSCrypt transports for protobuf and dnstap + + .. change:: + :tags: Bug Fixes + :pullreq: 10900 + + NetmaskTree: Drop the 'noexcept' qualifier on the TreeNode ctor + + .. change:: + :tags: Improvements + :pullreq: 10907 + :tickets: 4670 + + Handle existing EDNS content for SetMacAddrAction/SetEDNSOptionAction + + .. change:: + :tags: Bug Fixes, DNS over HTTPS, DNS over TLS + :pullreq: 10920 + + Fix the cleaning of TCP, DoT and DoH connections to the backend + + .. change:: + :tags: Bug Fixes + :pullreq: 10922 + + Fix build without nghttp2 + + .. change:: + :tags: New Features + :pullreq: 10923 + + Add the ability to retain select capabilities at runtime + + .. change:: + :tags: Bug Fixes + :pullreq: 10935 + + Remove debug print line flooding logs (Eugen Mayer) + + .. change:: + :tags: Bug Fixes + :pullreq: 10943 + :tickets: 10938 + + Credentials: EVP_PKEY_CTX_set1_scrypt_salt() takes an `unsigned char*` + + .. change:: + :tags: New Features, Performance + :pullreq: 10883, 10498 + + Implement filesystem pinning for eBPF maps, drop and truncate via XDP (Pierre Grié) + +.. changelog:: + :version: 1.7.0-alpha2 + :released: 19th of October 2021 + + .. change:: + :tags: Improvements + :pullreq: 10760 + + Don't create SSLKEYLOGFILE files with wide permissions + + .. change:: + :tags: Improvements + :pullreq: 10767 + + Update existing tags when calling setTagAction and setTagResponseAction + + .. change:: + :tags: Bug Fixes, DNS over HTTPS + :pullreq: 10772 + :tickets: 10771 + + Better handling of outgoing DoH workers + + .. change:: + :tags: Improvements + :pullreq: 10775 + :tickets: 10403 + + Fix the unit tests to handle v4-only or v6-only connectivity + + .. change:: + :tags: Improvements + :pullreq: 10782 + + Improve the coverage of the outgoing DoH code + + .. change:: + :tags: Bug Fixes + :pullreq: 10787 + + Properly cache UDP queries passed to a TCP/DoT/DoH backend + + .. change:: + :tags: Improvements + :pullreq: 10791 + + Allow skipping arbitrary EDNS options when computing packet hash + + .. change:: + :tags: New Features + :pullreq: 10814 + + Add lua support for SetEDNSOptionAction + + .. change:: + :tags: Improvements, DNS over HTTPS, DNS over TLS + :pullreq: 10823 + + Disable TLS renegotiation, release buffers for outgoing TLS + + .. change:: + :tags: New Features + :pullreq: 10832 + + Rule for basing decisions on outstanding queries in a pool (phonedph1) + + .. change:: + :tags: Improvements + :pullreq: 10833 + + Add incoming and outgoing protocols to grepq + + .. change:: + :tags: Improvements + :pullreq: 10835 + :tickets: 10559 + + Allow setting the block reason from the SMT callback + + .. change:: + :tags: Bug Fixes + :pullreq: 10841 + + Use per-thread credentials for GnuTLS client connections + + .. change:: + :tags: Improvements + :pullreq: 10844 + + Clear the UDP states of TCP-only backends + + .. change:: + :tags: Improvements + :pullreq: 10846 + + Replace shared by unique ptrs, reduce structs size + + .. change:: + :tags: Bug Fixes + :pullreq: 10848 + + Only set recursion protection once we know we do not return + +.. changelog:: + :version: 1.7.0-alpha1 + :released: 23rd of September 2021 + + .. change:: + :tags: Improvements + :pullreq: 10157 + :tickets: 7937 + + Move to hashed passwords for the web interface + + .. change:: + :tags: Improvements + :pullreq: 10381 + + Reorganize the IDState and Rings fields to reduce memory usage + + .. change:: + :tags: Improvements + :pullreq: 10429 + + Fix 'temporary used in loop' warnings reported by g++ 11.1.0 + + .. change:: + :tags: Improvements + :pullreq: 10441 + + Skip some memory allocations in client mode to reduce memory usage + + .. change:: + :tags: Improvements + :pullreq: 10414 + + Support multiple ip addresses for dnsdist-resolver lua script (Wim) + + .. change:: + :tags: Improvements + :pullreq: 10489 + :tickets: 10436 + + Make DNSDist XFR aware when transfer is finished (Dimitrios Mavrommatis) + + .. change:: + :tags: New Features + :pullreq: 10532 + :tickets: 10456 + + Add FFI functions to spoof multiple raw values + + .. change:: + :tags: Improvements + :pullreq: 10508 + :tickets: 10500 + + Do not report latency metrics of down upstream servers (Holger Hoffstätte) + + .. change:: + :tags: Improvements + :pullreq: 10537 + :tickets: 10338 + + Carry the exact incoming protocol (Do53, DNSCrypt, DoT, DoH) in DQ + + .. change:: + :tags: Improvements + :pullreq: 10527 + :tickets: 10502 + + Implement 'reload()' to rotate Log(Response)Action's log file + + .. change:: + :tags: New Features, Performance + :pullreq: 10501 + + Add support for Lua per-thread FFI rules and actions + + .. change:: + :tags: Improvements, Performance + :pullreq: 10520 + + Don't look up the LMDB dbi by name for every query + + .. change:: + :tags: New Features + :pullreq: 10525 + :tickets: 10520 + + Add support for range-based lookups into a Key-Value store + + .. change:: + :tags: Improvements + :pullreq: 10626 + + Document that setECSOverride has its drawbacks (Andreas Jakum) + + .. change:: + :tags: Improvements + :pullreq: 10649 + + Convert dnsdist and the recursor to LockGuarded + + .. change:: + :tags: Improvements + :pullreq: 10631 + + Handle waiting for a descriptor to become readable OR writable + + .. change:: + :tags: Bug Fixes + :pullreq: 10656 + + Catch FDMultiplexerException in IOStateHandler's destructor + + .. change:: + :tags: New Features, DNS over TLS + :pullreq: 10338 + + Implement cross-protocol queries, including outgoing DNS over TLS + + .. change:: + :tags: Bug Fixes + :pullreq: 10672 + + Resizing LMDB map size while there might be open transactions is unsafe + + .. change:: + :tags: New Features + :pullreq: 10597 + :tickets: 10367 + + Implement SpoofSVCAction to return SVC responses + + .. change:: + :tags: Bug Fixes + :pullreq: 10695 + :tickets: 10693 + + Ignore TCAction over TCP + + .. change:: + :tags: Improvements + :pullreq: 10687 + + Clean up a bit of "cast from type [...] casts away qualifiers" warnings + + .. change:: + :tags: New Features, DNS over HTTPS + :pullreq: 10635 + + Implementation of DoH between dnsdist and the backend + + .. change:: + :tags: Bug Fixes + :pullreq: 10704 + + Stop raising the number of TCP workers to the number of TCP binds + + .. change:: + :tags: Bug Fixes + :pullreq: 10724 + + Handle exception raised in IOStateGuard's destructor + +.. changelog:: + :version: 1.6.1 + :released: 15th of September 2021 + + .. change:: + :tags: Bug Fixes + :pullreq: 10438 + + Backport a missing mutex header + + .. change:: + :tags: Bug Fixes, DNSTAP + :pullreq: 10538 + :tickets: 10497 + + Set the dnstap/protobuf transport to TCP for DoH queries + + .. change:: + :tags: New Features + :pullreq: 10550 + :tickets: 10418 + + Add the missing DOHFronted::loadNewCertificatesAndKeys() + + .. change:: + :tags: New Features + :pullreq: 10560 + :tickets: 10482 + + Implement a web endpoint to get metrics for only one pool + + .. change:: + :tags: Bug Fixes + :pullreq: 10619 + :tickets: 10419 + + Properly handle ECS for queries with ancount or nscount > 0 + + .. change:: + :tags: Bug Fixes + :pullreq: 10656 + + Catch FDMultiplexerException in IOStateHandler's destructor + + .. change:: + :tags: Bug Fixes + :pullreq: 10706 + :tickets: 10705 + + Fix outstanding counter issue on TCP error + .. changelog:: :version: 1.6.0 :released: 11th of May 2021 @@ -2125,7 +2565,7 @@ Changelog :pullreq: 7481 :tickets: 6242 - Add support for encrypting ip addresses #gdpr + Add support for encrypting ip addresses #gdpr .. change:: :tags: Improvements diff --git a/pdns/dnsdistdist/docs/guides/dns-over-https.rst b/pdns/dnsdistdist/docs/guides/dns-over-https.rst index b4035ab0559f..78be8b0e31ae 100644 --- a/pdns/dnsdistdist/docs/guides/dns-over-https.rst +++ b/pdns/dnsdistdist/docs/guides/dns-over-https.rst @@ -1,9 +1,12 @@ DNS-over-HTTPS (DoH) ==================== -:program:`dnsdist` supports DNS-over-HTTPS (DoH, standardized in RFC 8484). +:program:`dnsdist` supports DNS-over-HTTPS (DoH, standardized in RFC 8484) for incoming queries since 1.4.0, and for outgoing queries since 1.7.0. To see if the installation supports this, run ``dnsdist --version``. -If the output shows ``dns-over-https(DOH)``, DNS-over-HTTPS is supported. +If the output shows ``dns-over-https(DOH)``, incoming DNS-over-HTTPS is supported. If ``outgoing-dns-over-https(nghttp2)`` shows up then outgoing DNS-over-HTTPS is supported. + +Incoming +-------- Adding a listen port for DNS-over-HTTPS can be done with the :func:`addDOHLocal` function, e.g.:: @@ -33,7 +36,7 @@ A particular attention should be taken to the permissions of the certificate and More information about sessions management can also be found in :doc:`../advanced/tls-sessions-management`. Custom responses ----------------- +^^^^^^^^^^^^^^^^ It is also possible to set HTTP response rules to intercept HTTP queries early, before the DNS payload, if any, has been processed, to send custom responses including error pages, redirects or even serve static content. First a rule needs to be defined using :func:`newDOHResponseMapEntry`, then a set of rules can be applied to a DoH frontend via :meth:`DOHFrontend:setResponsesMap`. For example, to send an HTTP redirect to queries asking for ``/rfc``, the following configuration can be used:: @@ -43,7 +46,7 @@ For example, to send an HTTP redirect to queries asking for ``/rfc``, the follow dohFE:setResponsesMap(map) DNS over HTTP -------------- +^^^^^^^^^^^^^ In case you want to run DNS-over-HTTPS behind a reverse proxy you probably don't want to encrypt your traffic between reverse proxy and dnsdist. To let dnsdist listen for DoH queries over HTTP on localhost at port 8053 add one of the following to your config:: @@ -52,7 +55,7 @@ To let dnsdist listen for DoH queries over HTTP on localhost at port 8053 add on addDOHLocal("127.0.0.1:8053", nil, nil, "/", { reusePort=true }) Internal design ---------------- +^^^^^^^^^^^^^^^ The internal design used for DoH handling uses two threads per :func:`addDOHLocal` directive. The first thread will handle the HTTP/2 communication with the client and pass the received DNS queries to a second thread which will apply the rules and pass the query to a backend, over **UDP** (except if the backend is TCP-only, or uses DNS over TLS, see the second schema below). The response will be received by the regular UDP response handler for that backend and passed back to the first thread. That allows the first thread to be low-latency dealing with TLS and HTTP/2 only and never blocking. @@ -68,10 +71,29 @@ Since 1.7.0, truncated answers received over UDP for a DoH query will lead to a :alt: DNSDist DoH design since 1.7 Investigating issues --------------------- +^^^^^^^^^^^^^^^^^^^^ dnsdist provides a lot of counters to investigate issues: * :func:`showTCPStats` will display a lot of information about current and passed connections * :func:`showTLSErrorCounters` some metrics about why TLS sessions failed to establish * :func:`showDOHResponseCodes` returns metrics about HTTP response codes sent by dnsdist + +Outgoing +-------- + +Support for securing the exchanges between dnsdist and the backend will be implemented in 1.7.0, and will lead to all queries, regardless of whether they were initially received by dnsdist over UDP, TCP, DoT or DoH, being forwarded over a secure DNS over HTTPS channel. +That support can be enabled via the ``dohPath`` parameter of the :func:`newServer` command. Additional parameters control the TLS provider used (``tls``), the validation of the certificate presented by the backend (``caStore``, ``validateCertificates``), the actual TLS ciphers used (``ciphers``, ``ciphersTLS13``) and the SNI value sent (``subjectName``). + + newServer({address="[2001:DB8::1]:443", tls="openssl", subjectName="doh.powerdns.com", dohPath="/dns-query", validateCertificates=true}) + + +Internal design +^^^^^^^^^^^^^^^ + +The incoming queries, after the processing of rules if any, are passed to one of the DoH workers over a pipe. The DoH worker handles the communication with the backend, retrieves the response, and either responds directly to the client (queries coming over UDP) or pass it back over a pipe to the initial thread (queries coming over TCP, DoT or DoH). +The number of outgoing DoH worker threads can be configured using :func:`setOutgoingDoHWorkerThreads`. + +.. figure:: ../imgs/DNSDistOutgoingDoH.png + :align: center + :alt: DNSDist outgoing DoH design diff --git a/pdns/dnsdistdist/docs/guides/dns-over-tls.rst b/pdns/dnsdistdist/docs/guides/dns-over-tls.rst index 3c964e22a305..91d95a5d22cd 100644 --- a/pdns/dnsdistdist/docs/guides/dns-over-tls.rst +++ b/pdns/dnsdistdist/docs/guides/dns-over-tls.rst @@ -30,6 +30,10 @@ Outgoing Support for securing the exchanges between dnsdist and the backend will be implemented in 1.7.0, and will lead to all queries, regardless of whether they were initially received by dnsdist over UDP, TCP, DoT or DoH, being forwarded over a secure DNS over TLS channel. That support can be enabled via the ``tls`` parameter of the :func:`newServer` command. Additional parameters control the validation of the certificate presented by the backend (``caStore``, ``validateCertificates``), the actual TLS ciphers used (``ciphers``, ``ciphersTLS13``) and the SNI value sent (``subjectName``). + + newServer({address="[2001:DB8::1]:853", tls="openssl", subjectName="dot.powerdns.com", validateCertificates=true}) + + Investigating issues -------------------- diff --git a/pdns/dnsdistdist/docs/guides/dynblocks.rst b/pdns/dnsdistdist/docs/guides/dynblocks.rst index 10e52d377718..a8958283c0b0 100644 --- a/pdns/dnsdistdist/docs/guides/dynblocks.rst +++ b/pdns/dnsdistdist/docs/guides/dynblocks.rst @@ -1,6 +1,12 @@ Dynamic Rule Generation ======================= +Dynamic Blocks can be seen are short-lived rules, automatically inserted based on configurable thresholds and the analysis of recently received traffic, and automatically removed after a configurable amount of time. + +The analyzed traffic is the one kept by dnsdist in its in-memory ring buffers. The number of entries kept in these ring buffers can be set via the :func:`setRingBuffersSize` directive, and the impact in terms of CPU and memory consumption is described in :doc:`../advanced/tuning`. + +That number of entries is crucial for the rate-based rules, like :func:`DynBlockRulesGroup:setQueryRate`, as they will never match if the number of entries in the ring buffer is too small for the required rate, as explained in more details below. + To set dynamic rules, based on recent traffic, define a function called :func:`maintenance` in Lua. It will get called every second, and from this function you can set rules to block traffic based on statistics. More exactly, the thread handling the :func:`maintenance` function will sleep for one second between each invocation, so if the function takes several seconds to complete it will not be invoked exactly every second. @@ -86,3 +92,22 @@ action is applied. dbr:setQueryRate(300, 10, "Exceeded query rate", 60, DNSAction.Drop, 100) Since 1.6.0, if a default eBPF filter has been set via :func:`setDefaultBPFFilter` dnsdist will automatically try to use it when a "drop" dynamic block is inserted via a :ref:`DynBlockRulesGroup`. eBPF blocks are applied in kernel space and are much more efficient than user space ones. Note that a regular block is also inserted so that any failure will result in a regular block being used instead of the eBPF one. + +Rate rules and size of the ring buffers +--------------------------------------- + +As explained in the introduction, the whole dynamic block feature is based on analyzing the recent traffic kept in dnsdist's in-memory ring buffers, whose content can be inspected via :func:`grepq`. + +The sizing of the buffers, in addition to having performance impacts explained in :doc:`../advanced/tuning`, directly impacts some of the dynamic block rules, like the rate and ratio-based ones. + +For example, if :func:`DynBlockRulesGroup:setQueryRate` is used to request the blocking for 60s of any client exceeding 1000 qps over 10s, like this: + +.. code-block:: lua + + dbr:setQueryRate(1000, 10, "Exceeded query rate", 60, DNSAction.Drop) + +For this rule to trigger, dnsdist will need to scan the ring buffers and find 1000 * 10 = 10000 queries, not older than 10s, from that client. Since a ring buffer has a fixed size, and new entries override the oldest ones when the buffer is full, that only works if there are enough entries in the buffer. + +This is even more obvious for the ratio-based rules, when they have a minimum number of responses set, because in that case they clearly require that number of responses to fit in the buffer. + +That requirement could be lifted a bit by the use of sampling, meaning that only one query out of 10 would be recorded, for example, and the total amount would be inferred from the queries present in the buffer. As of 1.7.0, sampling as unfortunately not been implemented yet. diff --git a/pdns/dnsdistdist/docs/guides/webserver.rst b/pdns/dnsdistdist/docs/guides/webserver.rst index d2f1e9699ecb..410950ff38b6 100755 --- a/pdns/dnsdistdist/docs/guides/webserver.rst +++ b/pdns/dnsdistdist/docs/guides/webserver.rst @@ -389,7 +389,7 @@ URL Endpoints .. http:get:: /api/v1/servers/localhost/pool?name=pool-name - .. versionadded:: 1.6.0 + .. versionadded:: 1.6.1 Get a quick overview of the pool named "pool-name". diff --git a/pdns/dnsdistdist/docs/imgs/DNSDistOutgoingDoH.png b/pdns/dnsdistdist/docs/imgs/DNSDistOutgoingDoH.png new file mode 100644 index 000000000000..7adb13effec3 Binary files /dev/null and b/pdns/dnsdistdist/docs/imgs/DNSDistOutgoingDoH.png differ diff --git a/pdns/dnsdistdist/docs/index.rst b/pdns/dnsdistdist/docs/index.rst index 3b8ede8174dc..faef45701e1d 100644 --- a/pdns/dnsdistdist/docs/index.rst +++ b/pdns/dnsdistdist/docs/index.rst @@ -10,7 +10,8 @@ A configuration to balance DNS queries to several backend servers: .. code-block:: lua - newServer({address="2620:fe::fe2620:fe::9", qps=1}) + newServer({address="2620:fe::fe", qps=1}) + newServer({address="2620:fe::9", qps=1}) newServer({address="9.9.9.9", qps=1}) newServer({address="2001:db8::1", qps=10}) newServer({address="[2001:db8::2]:5300", name="dns1", qps=10}) diff --git a/pdns/dnsdistdist/docs/install.rst b/pdns/dnsdistdist/docs/install.rst index fe299b4b1309..b622248004ac 100644 --- a/pdns/dnsdistdist/docs/install.rst +++ b/pdns/dnsdistdist/docs/install.rst @@ -42,15 +42,24 @@ dnsdist is also available in `FreeBSD ports `_ * `Lua `_ 5.1+ or `LuaJit `_ * `Editline (libedit) `_ -* `libsodium `_ (optional) +* `libfstrm `_ (optional, dnstap support) +* `GnuTLS `_ (optional, DoT and outgoing DoH support) +* `libh2o `_ (optional, incoming DoH support) +* `libcap `_ (optional, capabilities support) +* `libsodium `_ (optional, DNSCrypt and console encryption support) +* `LMDB `_ (optional, LMDB support) +* `net-snmp `_ (optional, SNMP support) +* `nghttp2 `_ (optional, outgoing DoH support) +* `OpenSSL `_ (optional, DoT and DoH support) * `protobuf `_ (optional, not needed as of 1.6.0) * `re2 `_ (optional) +* `TinyCDB ` (optional, CDB support) Should :program:`dnsdist` be run on a system with systemd, it is highly recommended to have the systemd header files (``libsystemd-dev`` on Debian and ``systemd-devel`` on CentOS) diff --git a/pdns/dnsdistdist/docs/reference/config.rst b/pdns/dnsdistdist/docs/reference/config.rst index 2ad4828c8c78..b2386707f029 100644 --- a/pdns/dnsdistdist/docs/reference/config.rst +++ b/pdns/dnsdistdist/docs/reference/config.rst @@ -37,6 +37,15 @@ The ``.`` means ``order`` is a data member, while the ``:`` means ``addPool`` is Global configuration -------------------- +.. function:: addCapabilitiesToRetain(capabilities) + + .. versionadded:: 1.7.0 + + Accept a Linux capability as a string, or a list of these, to retain after startup so that privileged operations can still be performed at runtime. + Keeping ``CAP_BPF`` on kernel 5.8+ for example allows loading eBPF programs and altering eBPF maps at runtime even if the ``kernel.unprivileged_bpf_disabled`` sysctl is set. + Note that this does not grant the capabilities to the process, doing so might be done by running it as root which we don't advise, or by adding capabilities via the systemd unit file, for example. + Please also be aware that switching to a different user via ``--uid`` will still drop all capabilities. + .. function:: includeDirectory(path) Include configuration files from ``path``. @@ -290,6 +299,15 @@ Control Socket, Console and Webserver Webserver configuration ~~~~~~~~~~~~~~~~~~~~~~~ +.. function:: hashPassword(password [, workFactor]) + + .. versionadded:: 1.7.0 + + Hash the supplied password using a random salt, and returns a string that can be used with :func:`setWebserverConfig`. + + :param string - password: The password to hash + :param int - workFactor: The work factor to use for the hash function (currently scrypt), as a power of two. Default is 1024. + .. function:: webserver(listen_address [, password[, apikey[, custom_headers[, acl]]]]) .. versionchanged:: 1.5.0 @@ -325,18 +343,23 @@ Webserver configuration .. versionchanged:: 1.6.0 ``statsRequireAuthentication``, ``maxConcurrentConnections`` optional parameters added. + .. versionchanged:: 1.7.0 + The optional ``password`` and ``apiKey`` parameters now accept hashed passwords. + The optional ``hashPlaintextCredentials`` parameter has been added. + Setup webserver configuration. See :func:`webserver`. :param table options: A table with key: value pairs with webserver options. Options: - * ``password=newPassword``: string - Changes the API password - * ``apiKey=newKey``: string - Changes the API Key (set to an empty string do disable it) + * ``password=newPassword``: string - Set the password used to access the internal webserver. Since 1.7.0 the password should be hashed and salted via the :func:`hashPassword` command. + * ``apiKey=newKey``: string - Changes the API Key (set to an empty string do disable it). Since 1.7.0 the key should be hashed and salted via the :func:`hashPassword` command. * ``custom_headers={[str]=str,...}``: map of string - Allows setting custom headers and removing the defaults. * ``acl=newACL``: string - List of IP addresses, as a string, that are allowed to open a connection to the web server. Defaults to "127.0.0.1, ::1". * ``statsRequireAuthentication``: bool - Whether access to the statistics (/metrics and /jsonstat endpoints) require a valid password or API key. Defaults to true. * ``maxConcurrentConnections``: int - The maximum number of concurrent web connections, or 0 which means an unlimited number. Defaults to 100. + * ``hashPlaintextCredentials``: bool - Whether passwords and API keys provided in plaintext should be hashed during startup, to prevent the plaintext versions from staying in memory. Doing so increases significantly the cost of verifying credentials. Defaults to false. .. function:: registerWebHandler(path, handler) @@ -489,7 +512,7 @@ Servers Added ``maxInFlight`` to server_table. .. versionchanged:: 1.7.0 - Added ``caStore``, ``checkTCP``, ``ciphers``, ``ciphers13``, ``subjectName``, ``tcpOnly``, ``tls`` and ``validateCertificates`` to server_table. + Added ``addXForwardedHeaders``, ``caStore``, ``checkTCP``, ``ciphers``, ``ciphers13``, ``dohPath``, ``enableRenegotiation``, ``releaseBuffers``, ``subjectName``, ``tcpOnly``, ``tls`` and ``validateCertificates`` to server_table. Add a new backend server. Call this function with either a string:: @@ -531,7 +554,7 @@ Servers -- "address@interface", e.g. "192.0.2.2@eth0" addXPF=NUM, -- Add the client's IP address and port to the query, along with the original destination address and port, -- using the experimental XPF record from `draft-bellis-dnsop-xpf `_ and the specified option code. Default is disabled (0) - sockets=NUM, -- Number of sockets (and thus source ports) used toward the backend server, defaults to a single one. Note that for backends which are multithreaded, this setting will have an effect on the number of cores that will be used to process traffic from dnsdist. For example you may want to set 'sockets' to a number somewhat higher than the number of worker threads configured in the backend, particularly if the Linux kernel is being used to distribute traffic to multiple threads listening on the same socket (via `reuseport`). + sockets=NUM, -- Number of UDP sockets (and thus source ports) used toward the backend server, defaults to a single one. Note that for backends which are multithreaded, this setting will have an effect on the number of cores that will be used to process traffic from dnsdist. For example you may want to set 'sockets' to a number somewhat higher than the number of worker threads configured in the backend, particularly if the Linux kernel is being used to distribute traffic to multiple threads listening on the same socket (via `reuseport`). disableZeroScope=BOOL, -- Disable the EDNS Client Subnet 'zero scope' feature, which does a cache lookup for an answer valid for all subnets (ECS scope of 0) before adding ECS information to the query and doing the regular lookup. This requires the ``parseECS`` option of the corresponding cache to be set to true rise=NUM, -- Require NUM consecutive successful checks before declaring the backend up, default: 1 useProxyProtocol=BOOL, -- Add a proxy protocol header to the query, passing along the client's IP address and port along with the original destination address and port. Default is disabled. @@ -539,12 +562,16 @@ Servers maxInFlight=NUM, -- Maximum number of in-flight queries. The default is 0, which disables out-of-order processing. It should only be enabled if the backend does support out-of-order processing. As of 1.6.0, out-of-order processing needs to be enabled on the frontend as well, via :func:`addLocal` and/or :func:`addTLSLocal`. Note that out-of-order is always enabled on DoH frontends. tcpOnly=BOOL, -- Always forward queries to that backend over TCP, never over UDP. Always enabled for TLS backends. Default is false. checkTCP=BOOL, -- Whether to do healthcheck queries over TCP, instead of UDP. Always enabled for DNS over TLS backend. Default is false. - tls=STRING, -- Enable DNS over TLS communications for this backend, using the TLS provider ("openssl" or "gnutls") passed in parameter. Default is an empty string, which means this backend is used for plain UDP and TCP. + tls=STRING, -- Enable DNS over TLS communications for this backend, or DNS over HTTPS if ``dohPath`` is set, using the TLS provider ("openssl" or "gnutls") passed in parameter. Default is an empty string, which means this backend is used for plain UDP and TCP. caStore=STRING, -- Specifies the path to the CA certificate file, in PEM format, to use to check the certificate presented by the backend. Default is an empty string, which means to use the system CA store. Note that this directive is only used if ``validateCertificates`` is set. ciphers=STRING, -- The TLS ciphers to use. The exact format depends on the provider used. When the OpenSSL provider is used, ciphers for TLS 1.3 must be specified via ``ciphersTLS13``. ciphersTLS13=STRING, -- The ciphers to use for TLS 1.3, when the OpenSSL provider is used. When the GnuTLS provider is used, ``ciphers`` applies regardless of the TLS protocol and this setting is not used. subjectName=STRING, -- The subject name passed in the SNI value of the TLS handshake, and against which to validate the certificate presented by the backend. Default is empty. - validateCertificates=BOOL -- Whether the certificate presented by the backend should be validated against the CA store (see ``caStore``). Default is true. + validateCertificates=BOOL,-- Whether the certificate presented by the backend should be validated against the CA store (see ``caStore``). Default is true. + dohPath=STRING, -- Enable DNS over HTTPS communication for this backend, using POST queries to the HTTP host supplied as ``subjectName`` and the HTTP path supplied in this parameter. + addXForwardedHeaders=BOOL,-- Whether to add X-Forwarded-For, X-Forwarded-Port and X-Forwarded-Proto headers to a DNS over HTTPS backend. + releaseBuffers=BOOL, -- Whether OpenSSL should release its I/O buffers when a connection goes idle, saving roughly 35 kB of memory per connection. Default to true. + enableRenegotiation=BOOL -- Whether secure TLS renegotiation should be enabled. Disabled by default since it increases the attack surface and is seldom used for DNS. }) :param str server_string: A simple IP:PORT string. @@ -762,6 +789,9 @@ See :doc:`../guides/cache` for a how to. ``cookieHashing`` parameter added. ``numberOfShards`` now defaults to 20. + .. versionchanged:: 1.7.0 + ``skipOptions`` parameter added. + Creates a new :class:`PacketCache` with the settings specified. :param int maxEntries: The maximum number of entries in this cache @@ -779,6 +809,7 @@ See :doc:`../guides/cache` for a how to. * ``staleTTL=60``: int - When the backend servers are not reachable, and global configuration ``setStaleCacheEntriesTTL`` is set appropriately, TTL that will be used when a stale cache entry is returned. * ``temporaryFailureTTL=60``: int - On a SERVFAIL or REFUSED from the backend, cache for this amount of seconds.. * ``cookieHashing=false``: bool - Whether EDNS Cookie values will be hashed, resulting in separate entries for different cookies in the packet cache. This is required if the backend is sending answers with EDNS Cookies, otherwise a client might receive an answer with the wrong cookie. + * ``skipOptions={}``: Extra list of EDNS option codes to skip when hashing the packet (if ``cookieHashing`` above is false, EDNS cookie option number will already be added to this list). .. class:: PacketCache @@ -1236,6 +1267,21 @@ faster than the existing rules. Represents a group of dynamic block rules. + .. method:: DynBlockRulesGroup:setMasks(v4, v6, port) + + .. versionadded:: 1.7.0 + + Set the number of bits to keep in the IP address when inserting a block. The default is 32 for IPv4 and 128 for IPv6, meaning + that only the exact address is blocked, but in some scenarios it might make sense to block a whole /64 IPv6 range instead of a + single address, for example. + It is also possible to take the IPv4 UDP and TCP ports into account, for CGNAT deployments, by setting the number of bits of the port + to consider. For example passing 2 as the last parameter, which only makes sense if the previous parameters are respectively 32 + and 128, will split a given IP address into four port ranges: 0-16383, 16384-32767, 32768-49151 and 49152-65535. + + :param int v4: Number of bits to keep for IPv4 addresses. Default is 32 + :param int v6: Number of bits to keep for IPv6 addresses. Default is 128 + :param int port: Number of bits of port to consider over IPv4. Default is 0 meaning that the port is not taken into account + .. method:: DynBlockRulesGroup:setQueryRate(rate, seconds, reason, blockingTime [, action [, warningRate]]) Adds a query rate-limiting rule, equivalent to: @@ -1313,9 +1359,12 @@ faster than the existing rules. .. versionadded:: 1.4.0 + .. versionchanged:: 1.7.0 + This visitor function can now optionally return an additional string which will be set as the ``reason`` for the dynamic block. + Set a Lua visitor function that will be called for each label of every domain seen in queries and responses. The function receives a `StatNode` object representing the stats of the parent, a second one with the stats of the current label and one with the stats of the current node plus all its children. Note that this function will not be called if a FFI version has been set using :meth:`DynBlockRulesGroup:setSuffixMatchRuleFFI` - If the function returns true, the current label will be blocked according to the `seconds`, `reason`, `blockingTime` and `action` parameters. + If the function returns true, the current label will be blocked according to the `seconds`, `reason`, `blockingTime` and `action` parameters. Since 1.7.0, the function can return an additional string, in addition to the boolean, which will be set as the ``reason`` for the dynamic block. Selected domains can be excluded from this processing using the :meth:`DynBlockRulesGroup:excludeDomains` method. This replaces the existing :func:`addDynBlockSMT` function. @@ -1560,7 +1609,7 @@ DOHFrontend .. method:: DOHFrontend:loadNewCertificatesAndKeys(certFile(s), keyFile(s)) - .. versionadded:: 1.7.0 + .. versionadded:: 1.6.1 Create and switch to a new TLS context using the same options than were passed to the corresponding `addDOHLocal()` directive, but loading new certificates and keys from the selected files, replacing the existing ones. diff --git a/pdns/dnsdistdist/docs/reference/dq.rst b/pdns/dnsdistdist/docs/reference/dq.rst index 76a432a1efb1..7271f1397417 100644 --- a/pdns/dnsdistdist/docs/reference/dq.rst +++ b/pdns/dnsdistdist/docs/reference/dq.rst @@ -47,7 +47,7 @@ This state can be modified from the various hooks. .. attribute:: DNSQuestion.qtype QType (as an unsigned integer) of this question. - Can be compared against the pre-defined :ref:`constants ` like ``DNSQType.A``, DNSQType.AAAA``. + Can be compared against the pre-defined :ref:`constants ` like ``DNSQType.A``, ``DNSQType.AAAA``. .. attribute:: DNSQuestion.remoteaddr @@ -137,6 +137,21 @@ This state can be modified from the various hooks. :returns: The scheme of the DoH query, for example ``http`` or ``https`` + .. method:: DNSQuestion:getProtocol() -> string + + .. versionadded:: 1.7.0 + + Return the transport protocol this query was received over, as a string. The possible values are: + + * "Do53 UDP" + * "Do53 TCP" + * "DNSCrypt UDP" + * "DNSCrypt TCP" + * "DNS over TLS" + * "DNS over HTTPS" + + :returns: A string + .. method:: DNSQuestion:getProxyProtocolValues() -> table .. versionadded:: 1.6.0 @@ -221,16 +236,20 @@ This state can be modified from the various hooks. .. method:: DNSQuestion:setTag(key, value) - Set a tag into the DNSQuestion object. - This function will not overwrite an existing tag. If the tag already exists it will keep its original value. + .. versionchanged:: 1.7.0 + Prior to 1.7.0 calling :func:`DNSQuestion:setTag` would not overwrite an existing tag value if already set. + + Set a tag into the DNSQuestion object. Overwrites the value if any already exists. :param string key: The tag's key :param string value: The tag's value .. method:: DNSQuestion:setTagArray(tags) - Set an array of tags into the DNSQuestion object. - This function will not overwrite an existing tag. If the tag already exists it will keep its original value. + .. versionchanged:: 1.7.0 + Prior to 1.7.0 calling :func:`DNSQuestion:setTagArray` would not overwrite existing tag values if already set. + + Set an array of tags into the DNSQuestion object. Overwrites the values if any already exist. :param table tags: A table of tags, using strings as keys and values diff --git a/pdns/dnsdistdist/docs/reference/ebpf.rst b/pdns/dnsdistdist/docs/reference/ebpf.rst index 3b12f57d6ab5..5fc3da9f0e4a 100644 --- a/pdns/dnsdistdist/docs/reference/ebpf.rst +++ b/pdns/dnsdistdist/docs/reference/ebpf.rst @@ -14,13 +14,27 @@ These are all the functions, objects and methods related to the :doc:`../advance :param str msg: A message to display while inserting the block .. function:: newBPFFilter(maxV4, maxV6, maxQNames) -> BPFFilter + newBPFFilter(v4Parameters, v6Parameters, qnamesParameters) -> BPFFilter - Return a new eBPF socket filter with a maximum of maxV4 IPv4, maxV6 IPv6 and maxQNames qname entries in the block table. + .. versionchanged:: 1.7.0 + This function now supports a table for each parameters, and the ability to use pinned eBPF maps. + + Return a new eBPF socket filter with a maximum of maxV4 IPv4, maxV6 IPv6 and maxQNames qname entries in the block tables. + Maps can be pinned to a filesystem path, which makes their content persistent across restarts and allows external programs to read their content and to add new entries. dnsdist will try to load maps that are pinned to a filesystem path on startups, inheriting any existing entries, and fall back to creating them if they do not exist yet. Note that the user dnsdist is running under must have the right privileges to read and write to the given file, and to go through all the directories in the path leading to that file. The pinned path must be on a filesystem of type ``BPF``, usually below ``/sys/fs/bpf/``. :param int maxV4: Maximum number of IPv4 entries in this filter :param int maxV6: Maximum number of IPv6 entries in this filter :param int maxQNames: Maximum number of QName entries in this filter + :param table v4Params: A table of options for the IPv4 filter map, see below + :param table v6Params: A table of options for the IPv6 filter map, see below + :param table qnameParams: A table of options for the qnames filter map, see below + + Options: + + * ``maxItems``: int - The maximum number of entries in a given map. Default is 0 which will not allow any entry at all. + * ``pinnedPath``: str - The filesystem path this map should be pinned to. + .. function:: newDynBPFFilter(bpf) -> DynBPFFilter Return a new dynamic eBPF filter associated to a given BPF Filter. diff --git a/pdns/dnsdistdist/docs/reference/index.rst b/pdns/dnsdistdist/docs/reference/index.rst index 92ba86af655f..9bccb195c18a 100755 --- a/pdns/dnsdistdist/docs/reference/index.rst +++ b/pdns/dnsdistdist/docs/reference/index.rst @@ -24,3 +24,4 @@ These chapters contain extensive information on all functions and object availab kvs logging web + svc \ No newline at end of file diff --git a/pdns/dnsdistdist/docs/reference/netmaskgroup.rst b/pdns/dnsdistdist/docs/reference/netmaskgroup.rst index f42dd290aa8a..b6d19b97ac1e 100644 --- a/pdns/dnsdistdist/docs/reference/netmaskgroup.rst +++ b/pdns/dnsdistdist/docs/reference/netmaskgroup.rst @@ -15,7 +15,7 @@ NetmaskGroup Add one or more masks to the NMG. :param string mask: Add this mask, prefix with `!` to exclude this mask from matching. - :param table masks: Adds the keys of the table to the :class:`NetmaskGroup`. It should be a table whose keys are :class:`ComboAddress` objects and values are integers, as returned by `exceed*` functions. + :param table masks: Adds the keys of the table to the :class:`NetmaskGroup`. It should be a table whose keys are :class:`ComboAddress` objects and whose values are integers. The integer values of the table entries are ignored. The table is of the same type as the table returned by the `exceed*` functions. .. method:: NetmaskGroup:match(address) -> bool diff --git a/pdns/dnsdistdist/docs/reference/svc.rst b/pdns/dnsdistdist/docs/reference/svc.rst new file mode 100644 index 000000000000..a93143df6978 --- /dev/null +++ b/pdns/dnsdistdist/docs/reference/svc.rst @@ -0,0 +1,45 @@ +SVCRecordParameters +=================== + +.. function:: newSVCRecordParameters(priority, target[, SVCParams]) -> SVCRecordParameters + + .. versionadded:: 1.7.0 + + Returns a :class:`SVCRecordParameters` to use with :func:`SpoofSVCAction`. + + .. code-block:: Lua + + -- reply to SVCB queries for _dns.resolver.arpa. indicating DoT on port 853 of dot.powerdns.com. (192.0.2.1/2001:db8::1), DoH on https://doh.powerdns.com/dns-query (192.0.2.2/2001:db8::2) + local svc = { newSVCRecordParameters(1, "dot.powerdns.com.", { mandatory={"port"}, alpn={ "dot" }, noDefaultAlpn=true, port=853, ipv4hint={ "192.0.2.1" }, ipv6hint={ "2001:db8::1" } }), + newSVCRecordParameters(2, "doh.powerdns.com.", { mandatory={"port"}, alpn={ "h2" }, port=443, ipv4hint={ "192.0.2.2" }, ipv6hint={ "2001:db8::2" }, key42 = "/dns-query{?dns}" }) + } + addAction(AndRule{QTypeRule(64), QNameRule('_dns.resolver.arpa.')}, SpoofSVCAction(svc)) + -- reply with NODATA (NXDOMAIN would deny all types at that name and below, including SVC) for other types + addAction(QNameRule('_dns.resolver.arpa.'), NegativeAndSOAAction(false, '_dns.resolver.arpa.', 3600, 'fake.resolver.arpa.', 'fake.resolver.arpa.', 1, 1800, 900, 604800, 86400)) + + + :param int priority: The priority of this record. if more than one record is returned, they all should have different priorities. A priority of 0 indicates Alias mode and no other record should be present in the RRSet. + :param str target: A domain name indicating the target name. + :param table SVCParams: Optional table of additionals parameters. The key should be the name of the SVC parameter and will be used as the SvcParamKey, while the value depends on the key (see below) + + These SVCParams can be set:: + + { + mandatory={STRING}, -- The mandatory keys. the table of strings must be the key names (like "port" and "key998"). + alpn={STRING}, -- alpns for this record, like "doh" or "h2". + noDefaultAlpn=BOOL, -- When true, the no-default-alpn key is included in the record, false or absent means it does not exist in the record. + port=NUM, -- Port parameter to include. + ipv4hint={STRING}, -- IPv4 hints to include into the record. + ech=STRING, -- Encrypted Client Hello as a raw string (can include null bytes). + ipv6hint={STRING} -- IPv6 hints to include into the record. + } + + Any other parameters can be set by using the ``keyNNNN`` syntax and must use a raw string. Like this:: + + key776="hello\0world" + +.. class:: SVCRecordParameters + + .. versionadded:: 1.7.0 + + Represents Service Binding (SVCB, HTTPS) record parameters, which can be used with :func:`SpoofSVCAction`. diff --git a/pdns/dnsdistdist/docs/reference/tuning.rst b/pdns/dnsdistdist/docs/reference/tuning.rst index c8911c26a28e..90442f5b80c1 100644 --- a/pdns/dnsdistdist/docs/reference/tuning.rst +++ b/pdns/dnsdistdist/docs/reference/tuning.rst @@ -1,6 +1,29 @@ Tuning related functions ======================== +.. function:: setDoHDownstreamCleanupInterval(interval) + + .. versionadded:: 1.7.0 + + Set how often, in seconds, the outgoing DoH connections to backends of a given worker thread are scanned to expunge the ones that are no longer usable. The default is 60 so once per minute and per worker thread. + :param int interval: The interval in seconds. + +.. function:: setDoHDownstreamMaxIdleTime(max) + + .. versionadded:: 1.7.0 + + Set how long, in seconds, an outgoing DoH connection to a backend might stay idle before being closed. The default is 300 so 5 minutes. + + :param int max: The maximum time in seconds. + +.. function:: setMaxIdleDoHConnectionsPerDownstream(max) + + .. versionadded:: 1.7.0 + + Set the maximum number of inactive DoH connections to a backend cached by each DoH worker thread. These connections can be reused when a new query comes in, instead of having to establish a new connection. dnsdist regularly checks whether the other end has closed any cached connection, closing them in that case. + + :param int max: The maximum number of inactive connections to keep. Default is 10, so 10 connections per backend and per DoH worker thread. + .. function:: setMaxCachedTCPConnectionsPerDownstream(max) .. versionadded:: 1.6.0 @@ -13,12 +36,16 @@ Tuning related functions .. versionchanged:: 1.6.0 Before 1.6.0 the default value was 10. + .. versionchanged:: 1.7.0 + The default value has been set back to 10. Set the maximum of TCP client threads, handling TCP connections. Before 1.4.0 a TCP thread could only handle a single incoming TCP connection at a time, while after 1.4.0 it can handle a larger number of them simultaneously. - Since 1.6.0, the default value is at least 10 TCP workers, but might be more if there is more than 10 TCP listeners (added via :func:`addDNSCryptBind`, :func:`addLocal`, or :func:`addTLSLocal`). In that last case there will be as many TCP workers as TCP listeners. + Note that before 1.6.0 the TCP worker threads were created at runtime, adding a new thread when the existing ones seemed to struggle with the load, until the maximum number of threads had been reached. Starting with 1.6.0 the configured number of worker threads are immediately created at startup. - :param int num: + In 1.6.0 the default value was at least 10 TCP workers, but could be more if there is more than 10 TCP listeners (added via :func:`addDNSCryptBind`, :func:`addLocal`, or :func:`addTLSLocal`). In that last case there would have been as many TCP workers as TCP listeners. This led to issues in setups with a large number of TCP listeners and was therefore reverted back to 10 in 1.7.0. + + :param int num: The number of TCP worker threads. .. function:: setMaxTCPConnectionDuration(num) @@ -68,12 +95,35 @@ Tuning related functions :param int num: +.. function:: setOutgoingDoHWorkerThreads(num) + + .. versionadded:: 1.7.0 + + Set the number of worker threads to use for outgoing DoH. That number defaults to 0 but is automatically raised to 1 when DoH is enabled on at least one backend. + .. function:: setStaleCacheEntriesTTL(num) Allows using cache entries expired for at most n seconds when no backend available to answer for a query :param int num: +.. function:: setTCPDownstreamCleanupInterval(interval) + + .. versionadded:: 1.6.0 + + Set how often, in seconds, the outgoing TCP connections to backends of a given worker thread are scanned to expunge the ones that are no longer usable. The default is 60 so once per minute and per worker thread. + + :param int interval: The interval in seconds. + +.. function:: setDoHDownstreamMaxIdleTime(max) + + .. versionadded:: 1.7.0 + + Set how long, in seconds, an outgoing DoH connection to a backend might stay idle before being closed. The default is 300 so 5 minutes. + + :param int max: The maximum time in seconds. + + .. function:: setTCPInternalPipeBufferSize(size) .. versionadded:: 1.6.0 diff --git a/pdns/dnsdistdist/docs/requirements.txt b/pdns/dnsdistdist/docs/requirements.txt index 63606d88e69a..5e3da903a230 100644 --- a/pdns/dnsdistdist/docs/requirements.txt +++ b/pdns/dnsdistdist/docs/requirements.txt @@ -4,4 +4,5 @@ git+https://github.com/pieterlexis/sphinx-jsondomain@no-type-links git+https://github.com/pieterlexis/sphinx-changelog@render-tags sphinxcontrib-httpdomain sphinxcontrib-fulltoc -docutils!=0.15 +docutils!=0.15,<0.18 +jinja2<3.1.0 diff --git a/pdns/dnsdistdist/docs/rules-actions.rst b/pdns/dnsdistdist/docs/rules-actions.rst index a76bf8368b57..fe322f52ff79 100644 --- a/pdns/dnsdistdist/docs/rules-actions.rst +++ b/pdns/dnsdistdist/docs/rules-actions.rst @@ -47,10 +47,13 @@ If this is not enough, try:: or:: - addAction(MaxQPSIPRule(5), TCAction()) + addAction(AndRule{MaxQPSIPRule(5), TCPRule(false)}, TCAction()) This will respectively drop traffic exceeding that 5 QPS limit per IP or range, or return it with TC=1, forcing clients to fall back to TCP. +In that last one, note the use of :func:`TCPRule`. +Without it, clients would get TC=1 even if they correctly fell back to TCP. + To turn this per IP or range limit into a global limit, use ``NotRule(MaxQPSRule(5000))`` instead of :func:`MaxQPSIPRule`. Regular Expressions @@ -69,6 +72,9 @@ The regex is applied case insensitively. Alternatively, if compiled in, :func:`RE2Rule` provides similar functionality, but against libre2. +Note that to check if a name is in a list of domains, :func:`SuffixMatchNodeRule` is preferred over complex regular expressions or multiple instances of :func:`RegexRule`. +The :func:`makeRule` convenience function can be used to create a :func:`SuffixMatchNodeRule`. + Rule Generators --------------- @@ -758,6 +764,21 @@ These ``DNSRule``\ s be one of the following items: :param string poolname: Pool to check +.. function:: PoolOutstandingRule(poolname, limit) + + .. versionadded:: 1.7.0 + + Check whether a pool has total outstanding queries above limit + + .. code-block:: Lua + + --- Send queries to spill over pool if default pool is under pressure + addAction(PoolOutstandingRule("", 5000), PoolAction("spillover")) + + :param string poolname: Pool to check + :param int limit: Total outstanding limit + + Combining Rules ~~~~~~~~~~~~~~~ @@ -1286,6 +1307,16 @@ The following actions exist. :param int v4: The IPv4 netmask length :param int v6: The IPv6 netmask length +.. function:: SetEDNSOptionAction(option) + + .. versionadded:: 1.7.0 + + Add arbitrary EDNS option and data to the query. Any existing EDNS content with the same option code will be overwritten. + Subsequent rules are processed after this action. + + :param int option: The EDNS option number + :param string data: The EDNS0 option raw content + .. function:: SetMacAddrAction(option) .. versionadded:: 1.6.0 @@ -1361,8 +1392,11 @@ The following actions exist. .. versionadded:: 1.6.0 + .. versionchanged:: 1.7.0 + Prior to 1.7.0 :func:`SetTagAction` would not overwrite an existing tag value if already set. + Associate a tag named ``name`` with a value of ``value`` to this query, that will be passed on to the response. - This function will not overwrite an existing tag. If the tag already exists it will keep its original value. + This function will overwrite any existing tag value. Subsequent rules are processed after this action. Note that this function was called :func:`TagAction` before 1.6.0. @@ -1373,8 +1407,11 @@ The following actions exist. .. versionadded:: 1.6.0 + .. versionchanged:: 1.7.0 + Prior to 1.7.0 :func:`SetTagResponseAction` would not overwrite an existing tag value if already set. + Associate a tag named ``name`` with a value of ``value`` to this response. - This function will not overwrite an existing tag. If the tag already exists it will keep its original value. + This function will overwrite any existing tag value. Subsequent rules are processed after this action. Note that this function was called :func:`TagResponseAction` before 1.6.0. @@ -1499,6 +1536,24 @@ The following actions exist. * ``ra``: bool - Set the RA bit to this value (true means the bit is set, false means it's cleared). Default is to copy the value of the RD bit from the incoming query. * ``ttl``: int - The TTL of the record. +.. function:: SpoofSVCAction(svcParams [, options]) + + .. versionadded:: 1.7.0 + + Forge a response with the specified SVC record data. If the list contains more than one class:`SVCRecordParameters` (generated via :func:`newSVCRecordParameters`) object, they are all returned, + and should have different priorities. + The hints provided in the SVC parameters, if any, will also be added as A/AAAA records in the additional section, using the target name present in the parameters as owner name if it's not empty (root) and the qname instead. + + :param list of class:`SVCRecordParameters` svcParams: The record data to return + :param table options: A table with key: value pairs with options. + + Options: + + * ``aa``: bool - Set the AA bit to this value (true means the bit is set, false means it's cleared). Default is to clear it. + * ``ad``: bool - Set the AD bit to this value (true means the bit is set, false means it's cleared). Default is to clear it. + * ``ra``: bool - Set the RA bit to this value (true means the bit is set, false means it's cleared). Default is to copy the value of the RD bit from the incoming query. + * ``ttl``: int - The TTL of the record. + .. function:: TagAction(name, value) .. deprecated:: 1.6.0 @@ -1523,7 +1578,12 @@ The following actions exist. .. function:: TCAction() + .. versionchanged:: 1.7.0 + This action is now only performed over UDP transports. + Create answer to query with the TC bit set, and the RA bit set to the value of RD in the query, to force the client to TCP. + Before 1.7.0 this action was performed even when the query had been received over TCP, which required the use of :func:`TCPRule` to + prevent the TC bit from being set over TCP transports. .. function:: TeeAction(remote[, addECS]) diff --git a/pdns/dnsdistdist/docs/running.rst b/pdns/dnsdistdist/docs/running.rst index 19d29ba98341..6d753dd6e2d9 100644 --- a/pdns/dnsdistdist/docs/running.rst +++ b/pdns/dnsdistdist/docs/running.rst @@ -38,25 +38,26 @@ Initially dnsdist tried to forward a query to the backend using the same protoco Before 1.7.0, which introduced TCP fallback, that meant that there was a potential issue with very large answers and DNS over HTTPS, requiring careful configuration of the path between dnsdist and the backend. More information about that is available in the :doc:`DNS over HTTPS section `. -In addition to TCP fallback for DoH, 1.7.0 introduced two new notions: +In addition to TCP fallback for DoH, 1.7.0 introduced three new notions: * TCP-only backends, for which queries will always forwarded over a TCP connection (see the `tcpOnly` parameter of :func:`newServer`) + * DNS over HTTPS backends, for which queries are forwarded over a DNS over HTTPS connection (see the `dohPath` parameter of :func:`newServer`) * and DNS over TLS backends, for which queries are forwarded over a DNS over TLS connection (see the `tls` parameter of :func:`newServer`) To sum it up: -+--------------+--------------------+---------------------------+----------------------+ -| Incoming | Outgoing (regular) | Outgoing (TCP-only, 1.7+) | Outgoing (TLS, 1.7+) | -+==============+====================+===========================+======================+ -| UDP | UDP | TCP | TLS | -+--------------+--------------------+---------------------------+----------------------+ -| TCP | TCP | TCP | TLS | -+--------------+--------------------+---------------------------+----------------------+ -| DNSCrypt UDP | UDP | TCP | TLS | -+--------------+--------------------+---------------------------+----------------------+ -| DNSCrypt TCP | TCP | TCP | TLS | -+--------------+--------------------+---------------------------+----------------------+ -| DoT | TCP | TCP | TLS | -+--------------+--------------------+---------------------------+----------------------+ -| DoH | **UDP** | TCP | TLS | -+--------------+--------------------+---------------------------+----------------------+ ++--------------+--------------------+---------------------------+----------------------+----------------------+ +| Incoming | Outgoing (regular) | Outgoing (TCP-only, 1.7+) | Outgoing (TLS, 1.7+) | Outgoing (DoH, 1.7+) | ++==============+====================+===========================+======================+======================+ +| UDP | UDP | TCP | TLS | DoH | ++--------------+--------------------+---------------------------+----------------------+----------------------+ +| TCP | TCP | TCP | TLS | DoH | ++--------------+--------------------+---------------------------+----------------------+----------------------+ +| DNSCrypt UDP | UDP | TCP | TLS | DoH | ++--------------+--------------------+---------------------------+----------------------+----------------------+ +| DNSCrypt TCP | TCP | TCP | TLS | DoH | ++--------------+--------------------+---------------------------+----------------------+----------------------+ +| DoT | TCP | TCP | TLS | DoH | ++--------------+--------------------+---------------------------+----------------------+----------------------+ +| DoH | **UDP** | TCP | TLS | DoH | ++--------------+--------------------+---------------------------+----------------------+----------------------+ diff --git a/pdns/dnsdistdist/docs/statistics.rst b/pdns/dnsdistdist/docs/statistics.rst index 194fdc8ce40e..66bc0184b6d1 100644 --- a/pdns/dnsdistdist/docs/statistics.rst +++ b/pdns/dnsdistdist/docs/statistics.rst @@ -237,6 +237,12 @@ trunc-failures -------------- Number of errors encountered while truncating an answer. +udp-in-csum-errors +------------------ +.. versionadded:: 1.7.0 + +From /proc/net/snmp InErrors. + udp-in-errors ------------- .. versionadded:: 1.5.0 @@ -261,6 +267,36 @@ udp-sndbuf-errors From /proc/net/snmp SndbufErrors. +udp6-in-csum-errors +------------------- +.. versionadded:: 1.7.0 + +From /proc/net/snmp6 InErrors. + +udp6-in-errors +-------------- +.. versionadded:: 1.7.0 + +From /proc/net/snmp6 InErrors. + +udp6-noport-errors +------------------ +.. versionadded:: 1.7.0 + +From /proc/net/snmp6 NoPorts. + +udp6-recvbuf-errors +------------------- +.. versionadded:: 1.7.0 + +From /proc/net/snmp6 RcvbufErrors. + +udp6-sndbuf-errors +------------------ +.. versionadded:: 1.7.0 + +From /proc/net/snmp6 SndbufErrors. + uptime ------ Uptime of the dnsdist process, in seconds. diff --git a/pdns/dnsdistdist/docs/upgrade_guide.rst b/pdns/dnsdistdist/docs/upgrade_guide.rst index 2dc77021d0e1..eb00fffddc6d 100644 --- a/pdns/dnsdistdist/docs/upgrade_guide.rst +++ b/pdns/dnsdistdist/docs/upgrade_guide.rst @@ -8,11 +8,19 @@ Truncated responses received over UDP for DoH clients will now be retried over T :func:`setTCPUseSinglePipe` has been removed. +Unless set via :func:`setMaxTCPClientThreads` the number of TCP workers now defaults to 10, instead of the number of TCP binds. + +Plain-text API keys and passwords for web server authentication are now strongly discouraged. The :func:`hashPassword` method can be used to generate a hashed and salted version of passwords and API keys instead, so that the plain-text version can no longer be found in either the configuration file or the memory of the running process. + 1.5.x to 1.6.0 -------------- The packet cache no longer hashes EDNS Cookies by default, which means that two queries that are identical except for the content of their cookie will now be served the same answer. This only works if the backend is not returning any answer containing EDNS Cookies, otherwise the wrong cookie might be returned to a client. To prevent this, the ``cookieHashing=true`` parameter might be passed to :func:`newPacketCache` so that cookies are hashed, resulting in separate entries in the packet cache. +All TCP worker threads are now created at startup, instead of being created on-demand. The existing behaviour was useful for very small setups but did not scale quickly to a large amount of TCP connections. +The new behaviour can cause a noticeable increase of TCP connections between dnsdist and its backends, as the TCP connections are not shared between TCP worker threads. +This is especially true for setups with a large number of frontends (:func:`addLocal`, :func:`addTLSLocal`, and :func:`addDNSCryptBind` directives), as 1.6.0 sets the number of TCP workers to the number of TCP-enabled binds (with a minimum of 10), unless that number has been set explicitely via :func:`setMaxTCPClientThreads`. + Several actions have been renamed so that almost all actions that allow further processing of rules start with 'Set', to prevent mistakes: - ``DisableECSAction`` to :func:`SetDisableECSAction` - ``DisableValidationAction`` to :func:`SetDisableValidationAction` diff --git a/pdns/dnsdistdist/doh.cc b/pdns/dnsdistdist/doh.cc index c479db666c81..3bfd66a00878 100644 --- a/pdns/dnsdistdist/doh.cc +++ b/pdns/dnsdistdist/doh.cc @@ -201,6 +201,13 @@ struct DOHServerConfig h2o_config_init(&h2o_config); h2o_config.http2.idle_timeout = idleTimeout * 1000; + /* if you came here for a way to make the number of concurrent streams (concurrent requests per connection) + configurable, or even just bigger, I have bad news for you. + h2o_config.http2.max_concurrent_requests_per_connection (default of 100) is capped by + H2O_HTTP2_SETTINGS_HOST.max_concurrent_streams which is not configurable. Even if decided to change the + hard-coded value, libh2o's author warns that there might be parts of the code where the stream ID is stored + in 8 bits, making 256 a hard value: https://github.com/h2o/h2o/issues/805 + */ } DOHServerConfig(const DOHServerConfig&) = delete; DOHServerConfig& operator=(const DOHServerConfig&) = delete; @@ -423,13 +430,13 @@ class DoHTCPCrossQuerySender : public TCPQuerySender return true; } - const ClientState& getClientState() override + const ClientState* getClientState() const override { if (!du || !du->dsc || !du->dsc->cs) { throw std::runtime_error("No query associated to this DoHTCPCrossQuerySender"); } - return *du->dsc->cs; + return du->dsc->cs; } void handleResponse(const struct timeval& now, TCPResponse&& response) override @@ -458,7 +465,7 @@ class DoHTCPCrossQuerySender : public TCPQuerySender double udiff = du->ids.sentTime.udiff(); vinfolog("Got answer from %s, relayed to %s (https), took %f usec", du->downstream->remote.toStringWithPort(), du->ids.origRemote.toStringWithPort(), udiff); - handleResponseSent(du->ids, udiff, *dr.remote, du->downstream->remote, du->response.size(), cleartextDH); + handleResponseSent(du->ids, udiff, *dr.remote, du->downstream->remote, du->response.size(), cleartextDH, du->downstream->getProtocol()); ++g_stats.responses; if (du->ids.cs) { @@ -485,6 +492,7 @@ class DoHTCPCrossQuerySender : public TCPQuerySender return; } + du->ids = std::move(query); du->status_code = 502; sendDoHUnitToTheMainThread(du, "cross-protocol error response"); du->release(); @@ -501,6 +509,11 @@ class DoHCrossProtocolQuery : public CrossProtocolQuery DoHCrossProtocolQuery(DOHUnit* du_): du(du_) { query = InternalQuery(std::move(du->query), std::move(du->ids)); + /* we _could_ remove it from the query buffer and put in query's d_proxyProtocolPayload, + clearing query.d_proxyProtocolPayloadAdded and du->proxyProtocolPayloadSize. + Leave it for now because we know that the onky case where the payload has been + added is when we tried over UDP, got a TC=1 answer and retried over TCP/DoT, + and we know the TCP/DoT code can handle it. */ query.d_proxyProtocolPayloadAdded = du->proxyProtocolPayloadSize > 0; downstream = du->downstream; proxyProtocolPayloadSize = du->proxyProtocolPayloadSize; @@ -541,7 +554,7 @@ static int processDOHQuery(DOHUnit* du) // outside of the main DoH thread return -1; } - remote = du->remote; + remote = du->ids.origRemote; DOHServerConfig* dsc = du->dsc; auto& holders = dsc->holders; ClientState& cs = *dsc->cs; @@ -586,8 +599,8 @@ static int processDOHQuery(DOHUnit* du) uint16_t qtype, qclass; unsigned int qnameWireLength = 0; DNSName qname(reinterpret_cast(du->query.data()), du->query.size(), sizeof(dnsheader), false, &qtype, &qclass, &qnameWireLength); - DNSQuestion dq(&qname, qtype, qclass, &du->dest, &du->remote, du->query, dnsdist::Protocol::DoH, &queryRealTime); - dq.ednsAdded = du->ednsAdded; + DNSQuestion dq(&qname, qtype, qclass, &du->ids.origDest, &du->ids.origRemote, du->query, dnsdist::Protocol::DoH, &queryRealTime); + dq.ednsAdded = du->ids.ednsAdded; dq.du = du; dq.sni = std::move(du->sni); @@ -618,25 +631,33 @@ static int processDOHQuery(DOHUnit* du) } if (du->downstream->isTCPOnly()) { - auto cpq = std::make_unique(du); + std::string proxyProtocolPayload; + /* we need to do this _before_ creating the cross protocol query because + after that the buffer will have been moved */ + if (du->downstream->useProxyProtocol) { + proxyProtocolPayload = getProxyProtocolPayload(dq); + } - du->get(); - du->tcp = true; du->ids.origID = htons(queryId); du->ids.cs = &cs; setIDStateFromDNSQuestion(du->ids, dq, std::move(qname)); - if (g_tcpclientthreads && g_tcpclientthreads->passCrossProtocolQueryToThread(std::move(cpq))) { + /* this moves du->ids, careful! */ + du->get(); + auto cpq = std::make_unique(du); + cpq->query.d_proxyProtocolPayload = std::move(proxyProtocolPayload); + du->tcp = true; + if (du->downstream->passCrossProtocolQuery(std::move(cpq))) { return 0; } else { - du->release(); + /* do not release du here, it belongs to the DoHCrossProtocolQuery object */ du->status_code = 502; return -1; } } - ComboAddress dest = du->dest; + ComboAddress dest = du->ids.origDest; unsigned int idOffset = (du->downstream->idOffset++) % du->downstream->idStates.size(); IDState* ids = &du->downstream->idStates[idOffset]; ids->age = 0; @@ -802,11 +823,11 @@ static void doh_dispatch_query(DOHServerConfig* dsc, h2o_handler_t* self, h2o_re uint16_t qtype; DNSName qname(reinterpret_cast(query.data()), query.size(), sizeof(dnsheader), false, &qtype); - auto du = std::unique_ptr(new DOHUnit); + auto du = std::make_unique(); du->dsc = dsc; du->req = req; - du->dest = local; - du->remote = remote; + du->ids.origDest = local; + du->ids.origRemote = remote; du->rsock = dsc->dohresponsepair[0]; du->query = std::move(query); du->path = std::move(path); @@ -820,8 +841,8 @@ static void doh_dispatch_query(DOHServerConfig* dsc, h2o_handler_t* self, h2o_re du->query_at = req->query_at; du->headers.reserve(req->headers.size); for (size_t i = 0; i < req->headers.size; ++i) { - du->headers.push_back(std::make_pair(std::string(req->headers.entries[i].name->base, req->headers.entries[i].name->len), - std::string(req->headers.entries[i].value.base, req->headers.entries[i].value.len))); + du->headers.emplace_back(std::string(req->headers.entries[i].name->base, req->headers.entries[i].name->len), + std::string(req->headers.entries[i].value.base, req->headers.entries[i].value.len)); } #ifdef HAVE_H2O_SOCKET_GET_SSL_SERVER_NAME @@ -947,16 +968,18 @@ static int doh_handler(h2o_handler_t *self, h2o_req_t *req) return 0; } - if (h2o_socket_get_ssl_session_reused(sock) == 0) { - ++dsc->cs->tlsNewSessions; - } - else { - ++dsc->cs->tlsResumptions; - } - const int descriptor = h2o_socket_get_fd(sock); if (descriptor != -1) { - ++t_conns.at(descriptor).d_nbQueries; + auto& conn = t_conns.at(descriptor); + ++conn.d_nbQueries; + if (conn.d_nbQueries == 1) { + if (h2o_socket_get_ssl_session_reused(sock) == 0) { + ++dsc->cs->tlsNewSessions; + } + else { + ++dsc->cs->tlsResumptions; + } + } } if (auto tlsversion = h2o_socket_get_ssl_protocol_version(sock)) { @@ -989,6 +1012,8 @@ static int doh_handler(h2o_handler_t *self, h2o_req_t *req) /* the responses map can be updated at runtime, so we need to take a copy of the shared pointer, increasing the reference counter */ auto responsesMap = dsc->df->d_responsesMap; + /* 1 byte for the root label, 2 type, 2 class, 4 TTL (fake), 2 record length, 2 option length, 2 option code, 2 family, 1 source, 1 scope, 16 max for a full v6 */ + const size_t maxAdditionalSizeForEDNS = 35U; if (responsesMap) { for (const auto& entry : *responsesMap) { if (entry->matches(path)) { @@ -1007,9 +1032,8 @@ static int doh_handler(h2o_handler_t *self, h2o_req_t *req) ++dsc->df->d_http1Stats.d_nbQueries; PacketBuffer query; - /* We reserve at least 512 additional bytes to be able to add EDNS, but we also want - at least s_maxPacketCacheEntrySize bytes to be able to fill the answer from the packet cache */ - query.reserve(std::max(req->entity.len + 512, s_maxPacketCacheEntrySize)); + /* We reserve a few additional bytes to be able to add EDNS later */ + query.reserve(req->entity.len + maxAdditionalSizeForEDNS); query.resize(req->entity.len); memcpy(query.data(), req->entity.base, req->entity.len); doh_dispatch_query(dsc, self, req, std::move(query), local, remote, std::move(path)); @@ -1036,10 +1060,9 @@ static int doh_handler(h2o_handler_t *self, h2o_req_t *req) PacketBuffer decoded; /* rough estimate so we hopefully don't need a new allocation later */ - /* We reserve at least 512 additional bytes to be able to add EDNS, but we also want - at least s_maxPacketCacheEntrySize bytes to be able to fill the answer from the packet cache */ + /* We reserve at few additional bytes to be able to add EDNS later */ const size_t estimate = ((sdns.size() * 3) / 4); - decoded.reserve(std::max(estimate + 512, s_maxPacketCacheEntrySize)); + decoded.reserve(estimate + maxAdditionalSizeForEDNS); if(B64Decode(sdns, decoded) < 0) { h2o_send_error_400(req, "Bad Request", "Unable to decode BASE64-URL", 0); ++dsc->df->d_badrequests; @@ -1237,7 +1260,7 @@ static void dnsdistclient(int qsock) if (generateOptRR(std::string(), du->query, 4096, 4096, 0, false)) { dh = const_cast(reinterpret_cast(du->query.data())); // may have reallocated dh->arcount = htons(1); - du->ednsAdded = true; + du->ids.ednsAdded = true; } } else { @@ -1270,53 +1293,65 @@ static void dnsdistclient(int qsock) */ static void on_dnsdist(h2o_socket_t *listener, const char *err) { - DOHUnit *du = nullptr; - DOHServerConfig* dsc = reinterpret_cast(listener->data); - ssize_t got = read(dsc->dohresponsepair[1], &du, sizeof(du)); - - if (got < 0) { - warnlog("Error reading a DOH internal response: %s", strerror(errno)); - return; - } - else if (static_cast(got) != sizeof(du)) { - return; - } + /* we want to read as many responses from the pipe as possible before + giving up. Even if we are overloaded and fighting with the DoH connections + for the CPU, the first thing we need to do is to send responses to free slots + anyway, otherwise queries and responses are piling up in our pipes, consuming + memory and likely coming up too late after the client has gone away */ + while (true) { + DOHUnit *du = nullptr; + DOHServerConfig* dsc = reinterpret_cast(listener->data); + ssize_t got = read(dsc->dohresponsepair[1], &du, sizeof(du)); + + if (got < 0) { + if (errno != EWOULDBLOCK && errno != EAGAIN) { + errlog("Error reading a DOH internal response: %s", strerror(errno)); + } + return; + } + else if (static_cast(got) != sizeof(du)) { + errlog("Error reading a DoH internal response, got %d bytes instead of the expected %d", got, sizeof(du)); + return; + } - if (!du->req) { // it got killed in flight - du->self = nullptr; - du->release(); - return; - } + if (!du->req) { // it got killed in flight + du->self = nullptr; + du->release(); + continue; + } - if (!du->tcp && du->truncated && du->response.size() > sizeof(dnsheader)) { - /* restoring the original ID */ - dnsheader* queryDH = reinterpret_cast(du->query.data() + du->proxyProtocolPayloadSize); - queryDH->id = du->ids.origID; + if (!du->tcp && du->truncated && du->response.size() > sizeof(dnsheader)) { + /* restoring the original ID */ + dnsheader* queryDH = reinterpret_cast(du->query.data() + du->proxyProtocolPayloadSize); + queryDH->id = du->ids.origID; - auto cpq = std::make_unique(du); + auto cpq = std::make_unique(du); - du->get(); - du->tcp = true; - du->truncated = false; + du->get(); + du->tcp = true; + du->truncated = false; - if (g_tcpclientthreads && g_tcpclientthreads->passCrossProtocolQueryToThread(std::move(cpq))) { - return; - } - else { - du->release(); + if (g_tcpclientthreads && g_tcpclientthreads->passCrossProtocolQueryToThread(std::move(cpq))) { + continue; + } + else { + du->release(); + vinfolog("Unable to pass DoH query to a TCP worker thread after getting a TC response over UDP"); + continue; + } } - } - if (du->self) { - // we are back in the h2o main thread now, so we don't risk - // a race (h2o killing the query) when accessing du->req anymore - *du->self = nullptr; // so we don't clean up again in on_generator_dispose - du->self = nullptr; - } + if (du->self) { + // we are back in the h2o main thread now, so we don't risk + // a race (h2o killing the query) when accessing du->req anymore + *du->self = nullptr; // so we don't clean up again in on_generator_dispose + du->self = nullptr; + } - handleResponse(*dsc->df, du->req, du->status_code, du->response, dsc->df->d_customResponseHeaders, du->contentType, true); + handleResponse(*dsc->df, du->req, du->status_code, du->response, dsc->df->d_customResponseHeaders, du->contentType, true); - du->release(); + du->release(); + } } /* called when a TCP connection has been accepted, the TLS session has not been established */ @@ -1346,8 +1381,8 @@ static void on_accept(h2o_socket_t *listener, const char *err) return; } - if (concurrentConnections > dsc->cs->tcpMaxConcurrentConnections) { - dsc->cs->tcpMaxConcurrentConnections = concurrentConnections; + if (concurrentConnections > dsc->cs->tcpMaxConcurrentConnections.load()) { + dsc->cs->tcpMaxConcurrentConnections.store(concurrentConnections); } auto& conn = t_conns[descriptor]; @@ -1417,7 +1452,7 @@ static void setupTLSContext(DOHAcceptContext& acceptCtx, auto ctx = libssl_init_server_context(tlsConfig, acceptCtx.d_ocspResponses); if (tlsConfig.d_enableTickets && tlsConfig.d_numberOfTicketsKeys > 0) { - acceptCtx.d_ticketKeys = std::unique_ptr(new OpenSSLTLSTicketKeysRing(tlsConfig.d_numberOfTicketsKeys)); + acceptCtx.d_ticketKeys = std::make_unique(tlsConfig.d_numberOfTicketsKeys); SSL_CTX_set_tlsext_ticket_key_cb(ctx.get(), &ticket_key_callback); libssl_set_ticket_key_callback_data(ctx.get(), &acceptCtx); } @@ -1454,7 +1489,7 @@ static void setupAcceptContext(DOHAcceptContext& ctx, DOHServerConfig& dsc, bool nativeCtx->hosts = dsc.h2o_config.hosts; ctx.d_ticketsKeyRotationDelay = dsc.df->d_tlsConfig.d_ticketsKeyRotationDelay; - if (setupTLS && !dsc.df->d_tlsConfig.d_certKeyPairs.empty()) { + if (setupTLS && dsc.df->isHTTPS()) { try { setupTLSContext(ctx, dsc.df->d_tlsConfig, @@ -1518,7 +1553,7 @@ void DOHFrontend::setup() d_dsc = std::make_shared(d_idleTimeout, d_internalPipeBufferSize); - if (!d_tlsConfig.d_certKeyPairs.empty()) { + if (isHTTPS()) { try { setupTLSContext(*d_dsc->accept_ctx, d_tlsConfig, @@ -1632,7 +1667,7 @@ void DOHUnit::handleUDPResponse(PacketBuffer&& udpResponse, IDState&& state) double udiff = ids.sentTime.udiff(); vinfolog("Got answer from %s, relayed to %s (https), took %f usec", downstream->remote.toStringWithPort(), ids.origRemote.toStringWithPort(), udiff); - handleResponseSent(ids, udiff, *dr.remote, downstream->remote, response.size(), cleartextDH); + handleResponseSent(ids, udiff, *dr.remote, downstream->remote, response.size(), cleartextDH, downstream->getProtocol()); ++g_stats.responses; if (ids.cs) { diff --git a/pdns/dnsdistdist/m4/dnsdist_enable_doh.m4 b/pdns/dnsdistdist/m4/dnsdist_enable_doh.m4 index 4053d7a09447..876a21890f2b 100644 --- a/pdns/dnsdistdist/m4/dnsdist_enable_doh.m4 +++ b/pdns/dnsdistdist/m4/dnsdist_enable_doh.m4 @@ -1,7 +1,7 @@ AC_DEFUN([DNSDIST_ENABLE_DNS_OVER_HTTPS], [ - AC_MSG_CHECKING([whether to enable DNS over HTTPS (DoH) support]) + AC_MSG_CHECKING([whether to enable incoming DNS over HTTPS (DoH) support]) AC_ARG_ENABLE([dns-over-https], - AS_HELP_STRING([--enable-dns-over-https], [enable DNS over HTTPS (DoH) support (requires libh2o) @<:@default=no@:>@]), + AS_HELP_STRING([--enable-dns-over-https], [enable incoming DNS over HTTPS (DoH) support (requires libh2o) @<:@default=no@:>@]), [enable_dns_over_https=$enableval], [enable_dns_over_https=no] ) diff --git a/pdns/dnsdistdist/m4/pdns_with_nghttp2.m4 b/pdns/dnsdistdist/m4/pdns_with_nghttp2.m4 new file mode 100644 index 000000000000..aa6a5c2fb19f --- /dev/null +++ b/pdns/dnsdistdist/m4/pdns_with_nghttp2.m4 @@ -0,0 +1,32 @@ +AC_DEFUN([PDNS_WITH_NGHTTP2], [ + AC_MSG_CHECKING([whether we will be linking in nghttp2]) + HAVE_NGHTTP2=0 + AC_ARG_WITH([nghttp2], + AS_HELP_STRING([--with-nghttp2],[use nghttp2 @<:@default=auto@:>@]), + [with_nghttp2=$withval], + [with_nghttp2=auto], + ) + AC_MSG_RESULT([$with_nghttp2]) + + AS_IF([test "x$with_nghttp2" != "xno"], [ + AS_IF([test "x$with_nghttp2" = "xyes" -o "x$with_nghttp2" = "xauto"], [ + PKG_CHECK_MODULES([NGHTTP2], [libnghttp2], [ + [HAVE_NGHTTP2=1] + AC_DEFINE([HAVE_NGHTTP2], [1], [Define to 1 if you have nghttp2]) + save_CFLAGS=$CFLAGS + save_LIBS=$LIBS + CFLAGS="$NGHTTP2_CFLAGS $CFLAGS" + LIBS="$NGHTTP2_LIBS $LIBS" + CFLAGS=$save_CFLAGS + LIBS=$save_LIBS + + ], [ : ]) + ]) + ]) + AM_CONDITIONAL([HAVE_NGHTTP2], [test "x$NGHTTP2_LIBS" != "x"]) + AS_IF([test "x$with_nghttp2" = "xyes"], [ + AS_IF([test x"$NGHTTP2_LIBS" = "x"], [ + AC_MSG_ERROR([nghttp2 requested but libraries were not found]) + ]) + ]) +]) diff --git a/pdns/dnsdistdist/tcpiohandler-mplexer.hh b/pdns/dnsdistdist/tcpiohandler-mplexer.hh index c8d98b8b987c..d62ba7828b81 100644 --- a/pdns/dnsdistdist/tcpiohandler-mplexer.hh +++ b/pdns/dnsdistdist/tcpiohandler-mplexer.hh @@ -14,11 +14,11 @@ class IOStateHandler { public: - IOStateHandler(FDMultiplexer& mplexer, const int fd): d_mplexer(mplexer), d_fd(fd), d_currentState(IOState::Done) + IOStateHandler(FDMultiplexer& mplexer, const int fd): d_mplexer(mplexer), d_fd(fd) { } - IOStateHandler(FDMultiplexer& mplexer): d_mplexer(mplexer), d_fd(-1), d_currentState(IOState::Done) + IOStateHandler(FDMultiplexer& mplexer): d_mplexer(mplexer), d_fd(-1) { } @@ -36,9 +36,14 @@ public: } } - IOState getState() const + bool isWaitingForRead() const { - return d_currentState; + return d_isWaitingForRead; + } + + bool isWaitingForWrite() const + { + return d_isWaitingForWrite; } void setSocket(int fd) @@ -54,22 +59,66 @@ public: update(IOState::Done); } + std::string getState() const + { + std::string result("--"); + result.reserve(2); + if (isWaitingForRead()) { + result.at(0) = 'R'; + } + if (isWaitingForWrite()) { + result.at(1) = 'W'; + } + return result; + } + + void add(IOState iostate, FDMultiplexer::callbackfunc_t callback, FDMultiplexer::funcparam_t callbackData, boost::optional ttd) + { + DEBUGLOG("in "<<__PRETTY_FUNCTION__<<" for fd "< ttd = boost::none) { - DEBUGLOG("in "<<__PRETTY_FUNCTION__<<" for fd "<reset(); + try { + d_handler->reset(); + } + catch (const FDMultiplexerException& e) { + /* that should not happen, but an exception raised from a destructor would be bad so better + safe than sorry */ + } d_enabled = false; } } diff --git a/pdns/dnsdistdist/test-credentials_cc.cc b/pdns/dnsdistdist/test-credentials_cc.cc new file mode 120000 index 000000000000..8f713318b2e2 --- /dev/null +++ b/pdns/dnsdistdist/test-credentials_cc.cc @@ -0,0 +1 @@ +../test-credentials_cc.cc \ No newline at end of file diff --git a/pdns/dnsdistdist/test-dnsdist-connections-cache.cc b/pdns/dnsdistdist/test-dnsdist-connections-cache.cc new file mode 100644 index 000000000000..a59163b039f6 --- /dev/null +++ b/pdns/dnsdistdist/test-dnsdist-connections-cache.cc @@ -0,0 +1,234 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_NO_MAIN + +#include + +#include "dnsdist-tcp-downstream.hh" + +class MockupConnection +{ +public: + MockupConnection(const std::shared_ptr& ds, std::unique_ptr&, const struct timeval&, std::string&&) : + d_ds(ds) + { + } + + bool canBeReused() const + { + return d_reusable; + } + + bool isUsable() const + { + return d_usable; + } + + bool willBeReusable(bool) const + { + return d_reusable; + } + + void setReused() + { + } + + struct timeval getLastDataReceivedTime() const + { + return d_lastDataReceivedTime; + } + + bool isIdle() const + { + return d_idle; + } + + void stopIO() + { + } + + std::shared_ptr getDS() const + { + return d_ds; + } + + std::shared_ptr d_ds; + struct timeval d_lastDataReceivedTime + { + 0, 0 + }; + bool d_reusable{true}; + bool d_usable{true}; + bool d_idle{false}; +}; + +BOOST_AUTO_TEST_SUITE(test_dnsdist_connections_cache) + +BOOST_AUTO_TEST_CASE(test_ConnectionsCache) +{ + DownstreamConnectionsManager manager; + const size_t maxIdleConnPerDownstream = 5; + const uint16_t cleanupInterval = 1; + const uint16_t maxIdleTime = 5; + manager.setMaxIdleConnectionsPerDownstream(maxIdleConnPerDownstream); + manager.setCleanupInterval(cleanupInterval); + manager.setMaxIdleTime(maxIdleTime); + + auto mplexer = std::unique_ptr(FDMultiplexer::getMultiplexerSilent()); + auto downstream1 = std::make_shared(ComboAddress("192.0.2.1")); + auto downstream2 = std::make_shared(ComboAddress("192.0.2.2")); + struct timeval now; + gettimeofday(&now, nullptr); + + auto conn = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + BOOST_REQUIRE(conn != nullptr); + BOOST_CHECK_EQUAL(manager.count(), 1U); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 1U); + BOOST_CHECK_EQUAL(manager.getIdleCount(), 0U); + + /* since the connection can be reused, we should get the same one */ + { + auto conn1 = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + BOOST_CHECK(conn.get() == conn1.get()); + BOOST_CHECK_EQUAL(manager.count(), 1U); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 1U); + } + + /* if we mark it non-usable, we should get a new one */ + conn->d_usable = false; + auto conn2 = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + BOOST_CHECK(conn.get() != conn2.get()); + BOOST_CHECK_EQUAL(manager.count(), 2U); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 2U); + + /* since the second connection can be reused, we should get it */ + { + auto conn3 = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + BOOST_CHECK(conn3.get() == conn2.get()); + BOOST_CHECK_EQUAL(manager.count(), 2U); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 2U); + } + + /* different downstream so different connection */ + auto differentConn = manager.getConnectionToDownstream(mplexer, downstream2, now, std::string()); + BOOST_REQUIRE(differentConn != nullptr); + BOOST_CHECK(differentConn.get() != conn.get()); + BOOST_CHECK(differentConn.get() != conn2.get()); + BOOST_CHECK_EQUAL(manager.count(), 3U); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 3U); + { + /* but we should be able to reuse it */ + auto sameConn = manager.getConnectionToDownstream(mplexer, downstream2, now, std::string()); + BOOST_CHECK(sameConn.get() == differentConn.get()); + BOOST_CHECK_EQUAL(manager.count(), 3U); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 3U); + } + + struct timeval later = now; + later.tv_sec += cleanupInterval + 1; + + /* mark the second connection as no longer usable */ + conn2->d_usable = false; + /* first one as well but still fresh so it will not get checked */ + conn->d_usable = true; + conn->d_lastDataReceivedTime = later; + /* third one is usable but idle for too long */ + differentConn->d_idle = true; + differentConn->d_lastDataReceivedTime = later; + differentConn->d_lastDataReceivedTime.tv_sec -= (maxIdleTime + 1); + + /* we should not do an actual cleanup attempt since the last cleanup was done recently */ + manager.cleanupClosedConnections(now); + BOOST_CHECK_EQUAL(manager.count(), 3U); + + manager.cleanupClosedConnections(later); + BOOST_CHECK_EQUAL(manager.count(), 1U); + + /* mark the remaining conn as non-usable, to get new ones */ + conn->d_usable = false; + conn->d_lastDataReceivedTime.tv_sec = 0; + + std::vector> conns = {conn}; + while (conns.size() < maxIdleConnPerDownstream) { + auto newConn = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + newConn->d_usable = false; + conns.push_back(newConn); + BOOST_CHECK_EQUAL(manager.count(), conns.size()); + } + + /* if we add a new one, the oldest should NOT get expunged because they are all active ones! */ + auto newConn = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + BOOST_CHECK_GT(manager.count(), maxIdleConnPerDownstream); + + { + /* mark all connections as not usable anymore */ + for (auto& c : conns) { + c->d_usable = false; + } + + /* except the last one */ + newConn->d_usable = true; + + BOOST_CHECK_EQUAL(manager.count(), conns.size() + 1); + later.tv_sec += cleanupInterval + 1; + manager.cleanupClosedConnections(later); + BOOST_CHECK_EQUAL(manager.count(), 1U); + } + + conns.clear(); + auto cleared = manager.clear(); + BOOST_CHECK_EQUAL(cleared, 1U); + + /* add 10 actives connections */ + while (conns.size() < 10) { + newConn = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + newConn->d_usable = false; + conns.push_back(newConn); + BOOST_CHECK_EQUAL(manager.count(), conns.size()); + BOOST_CHECK_EQUAL(manager.getActiveCount(), conns.size()); + } + /* now we mark them as idle */ + for (auto& c : conns) { + /* use a different shared_ptr to make sure that the comparison is done on the actual raw pointer */ + auto shared = c; + shared->d_idle = true; + BOOST_CHECK(manager.moveToIdle(shared)); + } + BOOST_CHECK_EQUAL(manager.count(), maxIdleConnPerDownstream); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 0U); + BOOST_CHECK_EQUAL(manager.getIdleCount(), maxIdleConnPerDownstream); + + { + /* if we ask for a connection, one of these should become active and no longer idle */ + /* but first we need to mark them as usable again */ + for (const auto& c : conns) { + c->d_usable = true; + } + auto got = manager.getConnectionToDownstream(mplexer, downstream1, now, std::string()); + BOOST_CHECK_EQUAL(manager.count(), maxIdleConnPerDownstream); + BOOST_CHECK_EQUAL(manager.getActiveCount(), 1U); + BOOST_CHECK_EQUAL(manager.getIdleCount(), maxIdleConnPerDownstream - 1U); + } +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/pdns/dnsdistdist/test-dnsdistdynblocks_hh.cc b/pdns/dnsdistdist/test-dnsdistdynblocks_hh.cc index 374f8db2a750..58593ccbe579 100644 --- a/pdns/dnsdistdist/test-dnsdistdynblocks_hh.cc +++ b/pdns/dnsdistdist/test-dnsdistdynblocks_hh.cc @@ -9,7 +9,7 @@ #include "dnsdist-rings.hh" Rings g_rings; -GlobalStateHolder> g_dynblockNMG; +GlobalStateHolder> g_dynblockNMG; GlobalStateHolder> g_dynblockSMT; shared_ptr g_defaultBPFFilter{nullptr}; @@ -24,10 +24,12 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate) { ComboAddress backend("192.0.2.42"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; unsigned int responseTime = 0; struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; size_t numberOfSeconds = 10; size_t blockDuration = 60; @@ -49,10 +51,10 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); /* we do not care about the response during that test, but we want to make sure these do not interfere with the computation */ - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfQueries); BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -71,8 +73,8 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -103,8 +105,8 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate) { for (size_t idx = 0; idx < numberOfQueries; idx++) { struct timespec when = now; when.tv_sec -= (9 - timeIdx); - g_rings.insertQuery(when, requestor1, qname, qtype, size, dh); - g_rings.insertResponse(when, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertQuery(when, requestor1, qname, qtype, size, dh, protocol); + g_rings.insertResponse(when, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries * numberOfSeconds); @@ -149,6 +151,237 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate) { } } +BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate_RangeV6) { + /* Check that we correctly group IPv6 addresses from the same /64 subnet into the same + dynamic block entry, if instructed to do so */ + dnsheader dh; + memset(&dh, 0, sizeof(dh)); + DNSName qname("rings.powerdns.com."); + ComboAddress requestor1("2001:db8::1"); + ComboAddress backend("2001:0db8:ffff:ffff:ffff:ffff:ffff:ffff"); + uint16_t qtype = QType::AAAA; + uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; + unsigned int responseTime = 0; + struct timespec now; + gettime(&now); + NetmaskTree emptyNMG; + + size_t numberOfSeconds = 10; + size_t blockDuration = 60; + const auto action = DNSAction::Action::Drop; + const std::string reason = "Exceeded query rate"; + + DynBlockRulesGroup dbrg; + dbrg.setQuiet(true); + dbrg.setMasks(32, 64, 0); + + /* block above 50 qps for numberOfSeconds seconds, no warning */ + dbrg.setQueryRate(50, 0, numberOfSeconds, reason, blockDuration, action); + + { + /* insert 45 qps from a given client in the last 10s + this should not trigger the rule */ + size_t numberOfQueries = 45 * numberOfSeconds; + g_rings.clear(); + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), 0U); + g_dynblockNMG.setState(emptyNMG); + + for (size_t idx = 0; idx < numberOfQueries; idx++) { + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); + /* we do not care about the response during that test, but we want to make sure + these do not interfere with the computation */ + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); + } + BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfQueries); + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); + + dbrg.apply(now); + BOOST_CHECK_EQUAL(g_dynblockNMG.getLocal()->size(), 0U); + BOOST_CHECK(g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(requestor1, 128, 16)) == nullptr); + } + + { + /* insert just above 50 qps from several clients in the same /64 IPv6 range in the last 10s, + this should trigger the rule this time */ + size_t numberOfQueries = (50 * numberOfSeconds) + 1; + g_rings.clear(); + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), 0U); + g_dynblockNMG.setState(emptyNMG); + + for (size_t idx = 0; idx < numberOfQueries; idx++) { + ComboAddress requestor("2001:db8::" + std::to_string(idx)); + g_rings.insertQuery(now, requestor, qname, qtype, size, dh, protocol); + g_rings.insertResponse(now, requestor, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); + } + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); + + dbrg.apply(now); + BOOST_CHECK_EQUAL(g_dynblockNMG.getLocal()->size(), 1U); + + { + /* beginning of the range should be blocked */ + const auto& block = g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(requestor1, 128, 16))->second; + BOOST_CHECK_EQUAL(block.reason, reason); + BOOST_CHECK_EQUAL(static_cast(block.until.tv_sec), now.tv_sec + blockDuration); + BOOST_CHECK(block.domain.empty()); + BOOST_CHECK(block.action == action); + BOOST_CHECK_EQUAL(block.blocks, 0U); + BOOST_CHECK_EQUAL(block.warning, false); + } + + { + /* end of the range should be blocked as well */ + ComboAddress end("2001:0db8:0000:0000:ffff:ffff:ffff:ffff"); + const auto& block = g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(end, 128, 16))->second; + BOOST_CHECK_EQUAL(block.reason, reason); + BOOST_CHECK_EQUAL(static_cast(block.until.tv_sec), now.tv_sec + blockDuration); + BOOST_CHECK(block.domain.empty()); + BOOST_CHECK(block.action == action); + BOOST_CHECK_EQUAL(block.blocks, 0U); + BOOST_CHECK_EQUAL(block.warning, false); + } + + { + /* outside of the range should NOT */ + ComboAddress out("2001:0db8:0000:0001::0"); + BOOST_CHECK(g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(out, 128, 16)) == nullptr); + } + } +} + +BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate_V4Ports) { + /* Check that we correctly split IPv4 addresses based on port ranges, when instructed to do so */ + dnsheader dh; + memset(&dh, 0, sizeof(dh)); + DNSName qname("rings.powerdns.com."); + ComboAddress requestor1("192.0.2.1:42"); + ComboAddress backend("192.0.2.254"); + uint16_t qtype = QType::AAAA; + uint16_t size = 42; + unsigned int responseTime = 0; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; + struct timespec now; + gettime(&now); + NetmaskTree emptyNMG; + + size_t numberOfSeconds = 10; + size_t blockDuration = 60; + const auto action = DNSAction::Action::Drop; + const std::string reason = "Exceeded query rate"; + + DynBlockRulesGroup dbrg; + dbrg.setQuiet(true); + /* split v4 by ports using a /2 (0 - 16383, 16384 - 32767, 32768 - 49151, 49152 - 65535) */ + dbrg.setMasks(32, 128, 2); + + /* block above 50 qps for numberOfSeconds seconds, no warning */ + dbrg.setQueryRate(50, 0, numberOfSeconds, reason, blockDuration, action); + + { + /* insert 45 qps from a given client in the last 10s + this should not trigger the rule */ + size_t numberOfQueries = 45 * numberOfSeconds; + g_rings.clear(); + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), 0U); + g_dynblockNMG.setState(emptyNMG); + + for (size_t idx = 0; idx < numberOfQueries; idx++) { + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); + /* we do not care about the response during that test, but we want to make sure + these do not interfere with the computation */ + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); + } + BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfQueries); + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); + + dbrg.apply(now); + BOOST_CHECK_EQUAL(g_dynblockNMG.getLocal()->size(), 0U); + BOOST_CHECK(g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(requestor1, 128, 16)) == nullptr); + } + + { + /* insert just above 50 qps from several clients in the same IPv4 port range in the last 10s, + this should trigger the rule this time */ + size_t numberOfQueries = (50 * numberOfSeconds) + 1; + g_rings.clear(); + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), 0U); + g_dynblockNMG.setState(emptyNMG); + + for (size_t idx = 0; idx < numberOfQueries; idx++) { + ComboAddress requestor("192.0.2.1:" + std::to_string(idx)); + g_rings.insertQuery(now, requestor, qname, qtype, size, dh, protocol); + g_rings.insertResponse(now, requestor, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); + } + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); + + dbrg.apply(now); + BOOST_CHECK_EQUAL(g_dynblockNMG.getLocal()->size(), 1U); + + { + /* beginning of the port range should be blocked */ + const auto& block = g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(ComboAddress("192.0.2.1:0"), 32, 16))->second; + BOOST_CHECK_EQUAL(block.reason, reason); + BOOST_CHECK_EQUAL(static_cast(block.until.tv_sec), now.tv_sec + blockDuration); + BOOST_CHECK(block.domain.empty()); + BOOST_CHECK(block.action == action); + BOOST_CHECK_EQUAL(block.blocks, 0U); + BOOST_CHECK_EQUAL(block.warning, false); + } + + { + /* end of the range should be blocked as well */ + const auto& block = g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(ComboAddress("192.0.2.1:16383"), 32, 16))->second; + BOOST_CHECK_EQUAL(block.reason, reason); + BOOST_CHECK_EQUAL(static_cast(block.until.tv_sec), now.tv_sec + blockDuration); + BOOST_CHECK(block.domain.empty()); + BOOST_CHECK(block.action == action); + BOOST_CHECK_EQUAL(block.blocks, 0U); + BOOST_CHECK_EQUAL(block.warning, false); + } + + { + /* outside of the range should not */ + BOOST_CHECK(g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(ComboAddress("192.0.2.1:16384"), 32, 16)) == nullptr); + } + + /* we (again) insert just above 50 qps from several clients the same IPv4 port range, this should update the block which will + check by looking at the blocked counter */ + { + auto block = g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(ComboAddress("192.0.2.1:0"), 32, 16)); + BOOST_REQUIRE(block != nullptr); + BOOST_CHECK_EQUAL(block->second.blocks, 0U); + block->second.blocks = 42U; + } + + g_rings.clear(); + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), 0U); + + for (size_t idx = 0; idx < numberOfQueries; idx++) { + ComboAddress requestor("192.0.2.1:" + std::to_string(idx)); + g_rings.insertQuery(now, requestor, qname, qtype, size, dh, protocol); + g_rings.insertResponse(now, requestor, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); + } + BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); + + dbrg.apply(now); + + BOOST_CHECK_EQUAL(g_dynblockNMG.getLocal()->size(), 1U); + { + /* previous address/port should still be blocked */ + auto block = g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(ComboAddress("192.0.2.1:0"), 32, 16)); + BOOST_REQUIRE(block != nullptr); + BOOST_CHECK_EQUAL(block->second.blocks, 42U); + } + + /* but not a different one */ + BOOST_CHECK(g_dynblockNMG.getLocal()->lookup(AddressAndPortRange(ComboAddress("192.0.2.1:16384"), 32, 16)) == nullptr); + + } +} + BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate_responses) { /* check that the responses are not accounted as queries when a rcode rate rule is defined (sounds very specific but actually happened) */ @@ -160,10 +393,12 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate_responses) { ComboAddress backend("192.0.2.42"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; unsigned int responseTime = 0; struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; /* 100k entries, one shard */ g_rings.setCapacity(1000000, 1); @@ -195,10 +430,10 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QueryRate_responses) { struct timespec when = now; when.tv_sec -= (99 - timeIdx); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(when, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(when, requestor1, qname, qtype, size, dh, protocol); /* we do not care about the response during that test, but we want to make sure these do not interfere with the computation */ - g_rings.insertResponse(when, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(when, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfQueries * 100); @@ -217,9 +452,10 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QTypeRate) { ComboAddress requestor2("192.0.2.2"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; size_t numberOfSeconds = 10; size_t blockDuration = 60; @@ -241,7 +477,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QTypeRate) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -259,7 +495,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QTypeRate) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, QType::A, size, dh); + g_rings.insertQuery(now, requestor1, qname, QType::A, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -277,7 +513,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_QTypeRate) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -304,10 +540,11 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRate) { ComboAddress backend("192.0.2.42"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; unsigned int responseTime = 100 * 1000; /* 100ms */ struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; size_t numberOfSeconds = 10; size_t blockDuration = 60; @@ -331,7 +568,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRate) { dh.rcode = rcode; for (size_t idx = 0; idx < numberOfResponses; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfResponses); @@ -349,7 +586,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRate) { dh.rcode = RCode::FormErr; for (size_t idx = 0; idx < numberOfResponses; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfResponses); @@ -368,7 +605,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRate) { dh.rcode = rcode; for (size_t idx = 0; idx < numberOfResponses; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfResponses); @@ -395,10 +632,11 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRatio) { ComboAddress backend("192.0.2.42"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; unsigned int responseTime = 100 * 1000; /* 100ms */ struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; time_t numberOfSeconds = 10; unsigned int blockDuration = 60; @@ -421,11 +659,11 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRatio) { dh.rcode = rcode; for (size_t idx = 0; idx < 20; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } dh.rcode = RCode::NoError; for (size_t idx = 0; idx < 80; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), 100U); @@ -442,7 +680,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRatio) { dh.rcode = RCode::FormErr; for (size_t idx = 0; idx < 50; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), 50U); @@ -460,11 +698,11 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRatio) { dh.rcode = rcode; for (size_t idx = 0; idx < 21; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } dh.rcode = RCode::NoError; for (size_t idx = 0; idx < 79; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), 100U); @@ -490,11 +728,11 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_RCodeRatio) { dh.rcode = rcode; for (size_t idx = 0; idx < 11; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } dh.rcode = RCode::NoError; for (size_t idx = 0; idx < 39; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), 50U); @@ -512,10 +750,11 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_ResponseByteRate) { ComboAddress backend("192.0.2.42"); uint16_t qtype = QType::AAAA; uint16_t size = 100; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; unsigned int responseTime = 100 * 1000; /* 100ms */ struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; size_t numberOfSeconds = 10; size_t blockDuration = 60; @@ -539,7 +778,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_ResponseByteRate) { dh.rcode = rcode; for (size_t idx = 0; idx < numberOfResponses; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfResponses); @@ -557,7 +796,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_ResponseByteRate) { dh.rcode = rcode; for (size_t idx = 0; idx < numberOfResponses; idx++) { - g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend); + g_rings.insertResponse(now, requestor1, qname, qtype, responseTime, size, dh, backend, outgoingProtocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfResponseEntries(), numberOfResponses); @@ -583,9 +822,10 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Warning) { ComboAddress requestor2("192.0.2.2"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; size_t numberOfSeconds = 10; size_t blockDuration = 60; @@ -607,7 +847,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Warning) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -625,7 +865,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Warning) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -653,7 +893,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Warning) { BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), 0U); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -682,7 +922,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Warning) { BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), 0U); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -713,7 +953,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Warning) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries); @@ -741,9 +981,10 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Ranges) { ComboAddress requestor2("192.0.2.42"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; size_t numberOfSeconds = 10; size_t blockDuration = 60; @@ -769,8 +1010,8 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesGroup_Ranges) { g_dynblockNMG.setState(emptyNMG); for (size_t idx = 0; idx < numberOfQueries; idx++) { - g_rings.insertQuery(now, requestor1, qname, qtype, size, dh); - g_rings.insertQuery(now, requestor2, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); + g_rings.insertQuery(now, requestor2, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(g_rings.getNumberOfQueryEntries(), numberOfQueries * 2); @@ -795,9 +1036,11 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { DNSName qname("rings.powerdns.com."); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; struct timespec now; gettime(&now); - NetmaskTree emptyNMG; + NetmaskTree emptyNMG; SuffixMatchTree emptySMT; size_t numberOfSeconds = 10; @@ -821,7 +1064,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { */ for (size_t idx = 0; idx < 256; idx++) { const ComboAddress requestor("192.0.2." + std::to_string(idx)); - g_rings.insertQuery(now, requestor, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor, qname, qtype, size, dh, protocol); } /* we apply the rules, all clients should be blocked */ @@ -869,15 +1112,15 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { dbrg.setSuffixMatchRule(numberOfSeconds, reason, blockDuration, action, [](const StatNode& node, const StatNode::Stat& self, const StatNode::Stat& children) { if (self.queries > 0) { - return true; + return std::tuple>(true, boost::none); } - return false; + return std::tuple>(false, boost::none); }); /* insert one fake response for 255 DNS names */ const ComboAddress requestor("192.0.2.1"); for (size_t idx = 0; idx < 256; idx++) { - g_rings.insertResponse(now, requestor, DNSName(std::to_string(idx)) + qname, qtype, 1000 /*usec*/, size, dh, requestor /* backend, technically, but we don't care */); + g_rings.insertResponse(now, requestor, DNSName(std::to_string(idx)) + qname, qtype, 1000 /*usec*/, size, dh, requestor /* backend, technically, but we don't care */, outgoingProtocol); } /* we apply the rules, all suffixes should be blocked */ @@ -915,6 +1158,62 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { BOOST_CHECK(g_dynblockSMT.getLocal()->getNodes().empty()); } + { + /* === reset everything for SMT, this time we will check that we can override the 'reason' via the visitor function === */ + DynBlockRulesGroup dbrg; + dbrg.setQuiet(true); + g_rings.clear(); + g_dynblockNMG.setState(emptyNMG); + g_dynblockSMT.setState(emptySMT); + + dbrg.setSuffixMatchRule(numberOfSeconds, reason, blockDuration, action, [](const StatNode& node, const StatNode::Stat& self, const StatNode::Stat& children) { + if (self.queries > 0) { + return std::tuple>(true, "blocked for a different reason"); + } + return std::tuple>(false, boost::none); + }); + + /* insert one fake response for 255 DNS names */ + const ComboAddress requestor("192.0.2.1"); + for (size_t idx = 0; idx < 256; idx++) { + g_rings.insertResponse(now, requestor, DNSName(std::to_string(idx)) + qname, qtype, 1000 /*usec*/, size, dh, requestor /* backend, technically, but we don't care */, dnsdist::Protocol::DoUDP); + } + + /* we apply the rules, all suffixes should be blocked */ + dbrg.apply(now); + + for (size_t idx = 0; idx < 256; idx++) { + const DNSName name(DNSName(std::to_string(idx)) + qname); + const auto* block = g_dynblockSMT.getLocal()->lookup(name); + BOOST_REQUIRE(block != nullptr); + /* simulate that: + - 1.rings.powerdns.com. got 1 query + ... + - 255. does 255 queries + */ + block->blocks = idx; + } + + /* now we ask for the top 20 offenders for each reason */ + StopWatch sw; + sw.start(); + auto top = DynBlockMaintenance::getTopSuffixes(20); + BOOST_REQUIRE_EQUAL(top.size(), 1U); + auto suffixes = top.at("blocked for a different reason"); + BOOST_REQUIRE_EQUAL(suffixes.size(), 20U); + auto it = suffixes.begin(); + for (size_t idx = 236; idx < 256; idx++) { + BOOST_CHECK_EQUAL(it->first, (DNSName(std::to_string(idx)) + qname)); + BOOST_CHECK_EQUAL(it->second, idx); + ++it; + } + + struct timespec expired = now; + expired.tv_sec += blockDuration + 1; + DynBlockMaintenance::purgeExpired(expired); + BOOST_CHECK(g_dynblockSMT.getLocal()->getNodes().empty()); + } + #ifdef BENCH_DYNBLOCKS { /* now insert 1M names */ @@ -926,9 +1225,9 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { dbrg.setSuffixMatchRule(numberOfSeconds, reason, blockDuration, action, [](const StatNode& node, const StatNode::Stat& self, const StatNode::Stat& children) { if (self.queries > 0) { - return true; + return std::tuple>(true, boost::none); } - return false; + return std::tuple>(false, boost::none); }); bool done = false; @@ -937,7 +1236,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { for (size_t idxC = 0; !done && idxC < 256; idxC++) { for (size_t idxD = 0; !done && idxD < 256; idxD++) { const DNSName victim(std::to_string(idxB) + "." + std::to_string(idxC) + "." + std::to_string(idxD) + qname.toString()); - g_rings.insertResponse(now, requestor, victim, qtype, 1000 /*usec*/, size, dh, requestor /* backend, technically, but we don't care */); + g_rings.insertResponse(now, requestor, victim, qtype, 1000 /*usec*/, size, dh, requestor /* backend, technically, but we don't care */, outgoingProtocol); if (g_rings.getNumberOfQueryEntries() == 1000000) { done = true; break; @@ -955,12 +1254,15 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { sw.start(); auto top = DynBlockMaintenance::getTopSuffixes(20); cerr<<"scanned 1000000 entries in "<getNodes().size(), 0U); } #endif @@ -979,7 +1281,7 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { for (size_t idxC = 0; !done && idxC < 256; idxC++) { for (size_t idxD = 0; !done && idxD < 256; idxD++) { const ComboAddress requestor("192." + std::to_string(idxB) + "." + std::to_string(idxC) + "." + std::to_string(idxD)); - g_rings.insertQuery(now, requestor, qname, qtype, size, dh); + g_rings.insertQuery(now, requestor, qname, qtype, size, dh, protocol); if (g_rings.getNumberOfQueryEntries() == 1000000) { done = true; break; @@ -1009,4 +1311,168 @@ BOOST_AUTO_TEST_CASE(test_DynBlockRulesMetricsCache_GetTopN) { #endif } +BOOST_AUTO_TEST_CASE(test_NetmaskTree) { + NetmaskTree nmt; + BOOST_CHECK_EQUAL(nmt.empty(), true); + BOOST_CHECK_EQUAL(nmt.size(), 0U); + nmt.insert(AddressAndPortRange(ComboAddress("130.161.252.0"), 24, 0)).second = 0; + BOOST_CHECK_EQUAL(nmt.empty(), false); + BOOST_CHECK_EQUAL(nmt.size(), 1U); + nmt.insert(AddressAndPortRange(ComboAddress("130.161.0.0"), 16, 0)).second = 1; + BOOST_CHECK_EQUAL(nmt.size(), 2U); + nmt.insert(AddressAndPortRange(ComboAddress("130.0.0.0"), 8, 0)).second = 2; + BOOST_CHECK_EQUAL(nmt.size(), 3U); + + BOOST_CHECK(nmt.lookup(ComboAddress("213.244.168.210")) == nullptr); + auto found = nmt.lookup(ComboAddress("130.161.252.29")); + BOOST_REQUIRE(found); + BOOST_CHECK_EQUAL(found->second, 0); + found = nmt.lookup(ComboAddress("130.161.180.1")); + BOOST_CHECK(found); + BOOST_CHECK_EQUAL(found->second, 1); + + BOOST_CHECK_EQUAL(nmt.lookup(ComboAddress("130.255.255.255"))->second, 2); + BOOST_CHECK_EQUAL(nmt.lookup(ComboAddress("130.161.252.255"))->second, 0); + BOOST_CHECK_EQUAL(nmt.lookup(ComboAddress("130.161.253.255"))->second, 1); + BOOST_CHECK_EQUAL(nmt.lookup(AddressAndPortRange(ComboAddress("130.255.255.255"), 32, 16))->second, 2); + BOOST_CHECK_EQUAL(nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.255"), 32, 16))->second, 0); + BOOST_CHECK_EQUAL(nmt.lookup(AddressAndPortRange(ComboAddress("130.161.253.255"), 32, 16))->second, 1); + + found = nmt.lookup(ComboAddress("130.145.180.1")); + BOOST_CHECK(found); + BOOST_CHECK_EQUAL(found->second, 2); + + nmt.insert(AddressAndPortRange(ComboAddress("0.0.0.0"), 0, 0)).second = 3; + BOOST_CHECK_EQUAL(nmt.size(), 4U); + nmt.insert(AddressAndPortRange(ComboAddress("0.0.0.0"), 7, 0)).second = 4; + BOOST_CHECK_EQUAL(nmt.size(), 5U); + nmt.insert(AddressAndPortRange(ComboAddress("0.0.0.0"), 15, 0)).second = 5; + BOOST_CHECK_EQUAL(nmt.size(), 6U); + BOOST_CHECK_EQUAL(nmt.lookup(AddressAndPortRange(ComboAddress("0.0.0.0"), 0, 0))->second, 3); + BOOST_CHECK_EQUAL(nmt.lookup(AddressAndPortRange(ComboAddress("0.0.0.0"), 7, 0))->second, 4); + BOOST_CHECK_EQUAL(nmt.lookup(AddressAndPortRange(ComboAddress("0.0.0.0"), 15, 0))->second, 5); + BOOST_CHECK_EQUAL(nmt.lookup(AddressAndPortRange(ComboAddress("0.0.0.0"), 32, 0))->second, 5); + + nmt.clear(); + BOOST_CHECK_EQUAL(nmt.empty(), true); + BOOST_CHECK_EQUAL(nmt.size(), 0U); + BOOST_CHECK(!nmt.lookup(ComboAddress("130.161.180.1"))); + + nmt.insert(AddressAndPortRange(ComboAddress("::1"), 128, 0)).second = 1; + BOOST_CHECK_EQUAL(nmt.empty(), false); + BOOST_CHECK_EQUAL(nmt.size(), 1U); + nmt.insert(AddressAndPortRange(ComboAddress("::"), 0, 0)).second = 0; + BOOST_CHECK_EQUAL(nmt.size(), 2U); + nmt.insert(AddressAndPortRange(ComboAddress("fe80::"), 16, 0)).second = 2; + BOOST_CHECK_EQUAL(nmt.size(), 3U); + BOOST_CHECK(nmt.lookup(ComboAddress("130.161.253.255")) == nullptr); + BOOST_CHECK_EQUAL(nmt.lookup(ComboAddress("::2"))->second, 0); + BOOST_CHECK_EQUAL(nmt.lookup(ComboAddress("::ffff"))->second, 0); + BOOST_CHECK_EQUAL(nmt.lookup(ComboAddress("::1"))->second, 1); + BOOST_CHECK_EQUAL(nmt.lookup(ComboAddress("fe80::1"))->second, 2); +} + +BOOST_AUTO_TEST_CASE(test_NetmaskTreePort) { + { + /* exact port matching */ + NetmaskTree nmt; + BOOST_CHECK_EQUAL(nmt.empty(), true); + BOOST_CHECK_EQUAL(nmt.size(), 0U); + nmt.insert(AddressAndPortRange(ComboAddress("130.161.252.42:65534"), 32, 16)).second = 0; + BOOST_CHECK_EQUAL(nmt.empty(), false); + BOOST_CHECK_EQUAL(nmt.size(), 1U); + + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("213.244.168.210"), 32, 16)) == nullptr); + + auto found = nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:65534"), 32, 16)); + BOOST_CHECK(found != nullptr); + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:65533"), 32, 16)) == nullptr); + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:65535"), 32, 16)) == nullptr); + } + + { + /* /15 port matching */ + NetmaskTree nmt; + BOOST_CHECK_EQUAL(nmt.empty(), true); + BOOST_CHECK_EQUAL(nmt.size(), 0U); + nmt.insert(AddressAndPortRange(ComboAddress("130.161.252.42:0"), 32, 15)).second = 0; + BOOST_CHECK_EQUAL(nmt.empty(), false); + BOOST_CHECK_EQUAL(nmt.size(), 1U); + + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("213.244.168.210"), 32, 16)) == nullptr); + + auto found = nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:0"), 32, 16)); + BOOST_CHECK(found != nullptr); + + found = nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:1"), 32, 16)); + BOOST_CHECK(found != nullptr); + + /* everything else should be a miss */ + for (size_t idx = 2; idx <= 65535; idx++) { + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:" + std::to_string(idx)), 32, 16)) == nullptr); + } + + nmt.clear(); + BOOST_CHECK_EQUAL(nmt.empty(), true); + BOOST_CHECK_EQUAL(nmt.size(), 0U); + nmt.insert(AddressAndPortRange(ComboAddress("130.161.252.42:65535"), 32, 15)).second = 0; + BOOST_CHECK_EQUAL(nmt.empty(), false); + BOOST_CHECK_EQUAL(nmt.size(), 1U); + + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("213.244.168.210"), 32, 16)) == nullptr); + + /* everything else should be a miss */ + for (size_t idx = 0; idx <= 65533; idx++) { + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:" + std::to_string(idx)), 32, 16)) == nullptr); + } + found = nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:65534"), 32, 16)); + BOOST_CHECK(found != nullptr); + found = nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:65535"), 32, 16)); + BOOST_CHECK(found != nullptr); + } + + { + /* /1 port matching */ + NetmaskTree nmt; + BOOST_CHECK_EQUAL(nmt.empty(), true); + BOOST_CHECK_EQUAL(nmt.size(), 0U); + nmt.insert(AddressAndPortRange(ComboAddress("130.161.252.42:0"), 32, 1)).second = 0; + BOOST_CHECK_EQUAL(nmt.empty(), false); + BOOST_CHECK_EQUAL(nmt.size(), 1U); + + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("213.244.168.210"), 32, 16)) == nullptr); + + for (size_t idx = 0; idx <= 32767; idx++) { + auto found = nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:" + std::to_string(idx)), 32, 16)); + BOOST_CHECK(found != nullptr); + } + + /* everything else should be a miss */ + for (size_t idx = 32768; idx <= 65535; idx++) { + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("130.161.252.42:" + std::to_string(idx)), 32, 16)) == nullptr); + } + } + + { + /* Check that the port matching does not apply to IPv6, where it does not make sense */ + + /* /1 port matching */ + NetmaskTree nmt; + BOOST_CHECK_EQUAL(nmt.empty(), true); + BOOST_CHECK_EQUAL(nmt.size(), 0U); + nmt.insert(AddressAndPortRange(ComboAddress("[2001:db8::1]:0"), 128, 1)).second = 0; + BOOST_CHECK_EQUAL(nmt.empty(), false); + BOOST_CHECK_EQUAL(nmt.size(), 1U); + + /* different IP, no match */ + BOOST_CHECK(nmt.lookup(AddressAndPortRange(ComboAddress("[2001:db8::2]:0"), 128, 16)) == nullptr); + + /* all ports should match */ + for (size_t idx = 1; idx <= 65535; idx++) { + auto found = nmt.lookup(AddressAndPortRange(ComboAddress("[2001:db8::1]:" + std::to_string(idx)), 128, 16)); + BOOST_CHECK(found != nullptr); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/dnsdistdist/test-dnsdistkvs_cc.cc b/pdns/dnsdistdist/test-dnsdistkvs_cc.cc index bc013bd0892e..cc21b1f3b28f 100644 --- a/pdns/dnsdistdist/test-dnsdistkvs_cc.cc +++ b/pdns/dnsdistdist/test-dnsdistkvs_cc.cc @@ -354,11 +354,11 @@ BOOST_AUTO_TEST_CASE(test_LMDB) { transaction->commit(); } - auto lmdb = std::unique_ptr(new LMDBKVStore(dbPath, "db-name")); + std::unique_ptr lmdb = std::make_unique(dbPath, "db-name"); doKVSChecks(lmdb, lc, rem, dq, plaintextDomain); lmdb.reset(); - lmdb = std::unique_ptr(new LMDBKVStore(dbPath, "range-db-name")); + lmdb = std::make_unique(dbPath, "range-db-name"); doKVSRangeChecks(lmdb); /* std::string value; @@ -414,7 +414,7 @@ BOOST_AUTO_TEST_CASE(test_CDB) { writer.close(); } - auto cdb = std::unique_ptr(new CDBKVStore(db, 0)); + std::unique_ptr cdb = std::make_unique(db, 0); doKVSChecks(cdb, lc, rem, dq, plaintextDomain); /* diff --git a/pdns/dnsdistdist/test-dnsdistnghttp2_cc.cc b/pdns/dnsdistdist/test-dnsdistnghttp2_cc.cc new file mode 100644 index 000000000000..edea2c88df66 --- /dev/null +++ b/pdns/dnsdistdist/test-dnsdistnghttp2_cc.cc @@ -0,0 +1,1986 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_NO_MAIN + +#include + +#include "dnswriter.hh" +#include "dnsdist.hh" +#include "dnsdist-proxy-protocol.hh" +#include "dnsdist-rings.hh" +#include "dnsdist-nghttp2.hh" +#include "sstuff.hh" + +#ifdef HAVE_NGHTTP2 +#include + +BOOST_AUTO_TEST_SUITE(test_dnsdistnghttp2_cc) + +struct ExpectedStep +{ +public: + enum class ExpectedRequest + { + handshakeClient, + readFromClient, + writeToClient, + closeClient, + connectToBackend, + readFromBackend, + writeToBackend, + closeBackend + }; + + ExpectedStep(ExpectedRequest r, IOState n, size_t b = 0, std::function fn = nullptr) : + cb(fn), request(r), nextState(n), bytes(b) + { + } + + std::function cb{nullptr}; + ExpectedRequest request; + IOState nextState; + size_t bytes{0}; +}; + +struct ExpectedData +{ + PacketBuffer d_query; + PacketBuffer d_response; +}; + +static std::deque s_steps; +static std::map s_responses; + +std::ostream& operator<<(std::ostream& os, const ExpectedStep::ExpectedRequest d); + +std::ostream& operator<<(std::ostream& os, const ExpectedStep::ExpectedRequest d) +{ + static const std::vector requests = {"handshake with client", "read from client", "write to client", "close connection to client", "connect to the backend", "read from the backend", "write to the backend", "close connection to backend"}; + os << requests.at(static_cast(d)); + return os; +} + +class DOHConnection +{ +public: + DOHConnection(bool needProxyProtocol) : + d_session(std::unique_ptr(nullptr, nghttp2_session_del)), d_needProxyProtocol(needProxyProtocol) + { + nghttp2_session_callbacks* cbs = nullptr; + nghttp2_session_callbacks_new(&cbs); + std::unique_ptr callbacks(cbs, nghttp2_session_callbacks_del); + cbs = nullptr; + nghttp2_session_callbacks_set_send_callback(callbacks.get(), send_callback); + nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks.get(), on_frame_recv_callback); + nghttp2_session_callbacks_set_on_data_chunk_recv_callback(callbacks.get(), on_data_chunk_recv_callback); + nghttp2_session_callbacks_set_on_stream_close_callback(callbacks.get(), on_stream_close_callback); + nghttp2_session* sess = nullptr; + nghttp2_session_server_new(&sess, callbacks.get(), this); + d_session = std::unique_ptr(sess, nghttp2_session_del); + + nghttp2_settings_entry iv[1] = { + {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, 100}}; + nghttp2_submit_settings(d_session.get(), NGHTTP2_FLAG_NONE, iv, sizeof(iv) / sizeof(*iv)); + } + + PacketBuffer d_serverOutBuffer; + PacketBuffer d_proxyProtocolBuffer; + std::map d_queries; + std::map d_responses; + std::unique_ptr d_session; + /* used to replace the stream ID in outgoing frames. Ugly but the library does not let us + test weird cases without that */ + std::map d_idMapping; + bool d_needProxyProtocol; + + size_t submitIncoming(const PacketBuffer& data, size_t pos, size_t toWrite) + { + size_t consumed = 0; + if (d_needProxyProtocol) { + do { + auto bytesRemaining = isProxyHeaderComplete(d_proxyProtocolBuffer); + if (bytesRemaining < 0) { + size_t toConsume = toWrite > static_cast(-bytesRemaining) ? static_cast(-bytesRemaining) : toWrite; + d_proxyProtocolBuffer.insert(d_proxyProtocolBuffer.end(), data.begin() + pos, data.begin() + pos + toConsume); + pos += toConsume; + toWrite -= toConsume; + consumed += toConsume; + + bytesRemaining = isProxyHeaderComplete(d_proxyProtocolBuffer); + if (bytesRemaining > 0) { + d_needProxyProtocol = false; + } + else if (bytesRemaining == 0) { + throw("Fatal error while parsing proxy protocol payload"); + } + } + else if (bytesRemaining == 0) { + throw("Fatal error while parsing proxy protocol payload"); + } + + if (toWrite == 0) { + return consumed; + } + } while (d_needProxyProtocol && toWrite > 0); + } + + ssize_t readlen = nghttp2_session_mem_recv(d_session.get(), &data.at(pos), toWrite); + if (readlen < 0) { + throw("Fatal error while submitting: " + std::string(nghttp2_strerror(static_cast(readlen)))); + } + + /* just in case, see if we have anything to send */ + int rv = nghttp2_session_send(d_session.get()); + if (rv != 0) { + throw("Fatal error while sending: " + std::string(nghttp2_strerror(rv))); + } + + return readlen; + } + + void submitResponse(uint32_t streamId, PacketBuffer& data) + { + const nghttp2_nv hdrs[] = {{(uint8_t*)":status", (uint8_t*)"200", sizeof(":status") - 1, sizeof("200") - 1, NGHTTP2_NV_FLAG_NONE}}; + nghttp2_data_provider dataProvider; + dataProvider.source.ptr = &data; + dataProvider.read_callback = [](nghttp2_session* session, int32_t stream_id, uint8_t* buf, size_t length, uint32_t* data_flags, nghttp2_data_source* source, void* user_data) -> ssize_t { + auto buffer = reinterpret_cast(source->ptr); + size_t toCopy = 0; + if (buffer->size() > 0) { + toCopy = length > buffer->size() ? buffer->size() : length; + memcpy(buf, &buffer->at(0), toCopy); + buffer->erase(buffer->begin(), buffer->begin() + toCopy); + } + + if (buffer->size() == 0) { + *data_flags |= NGHTTP2_DATA_FLAG_EOF; + } + // cerr<<"submitting response data of size "<(user_data); + // cerr<<"inserting "<d_serverOutBuffer.size()<d_idMapping.empty() && length > 9) { + /* frame type == DATA */ + if (data[3] == NGHTTP2_DATA) { + uint32_t streamId = 0; + memcpy(&streamId, &data[5], sizeof(streamId)); + const auto it = conn->d_idMapping.find(ntohl(streamId)); + if (it != conn->d_idMapping.end()) { + streamId = htonl(it->second); + std::vector editedData(length); + std::copy(data, data + length, editedData.begin()); + memcpy(&editedData.at(5), &streamId, sizeof(streamId)); + conn->d_serverOutBuffer.insert(conn->d_serverOutBuffer.end(), editedData.data(), editedData.data() + length); + return static_cast(editedData.size()); + } + } + } + + conn->d_serverOutBuffer.insert(conn->d_serverOutBuffer.end(), data, data + length); + return static_cast(length); + } + + static int on_frame_recv_callback(nghttp2_session* session, const nghttp2_frame* frame, void* user_data) + { + DOHConnection* conn = reinterpret_cast(user_data); + // cerr<<"Frame type is "<hd.type)<hd.type == NGHTTP2_HEADERS || frame->hd.type == NGHTTP2_DATA) && frame->hd.flags & NGHTTP2_FLAG_END_STREAM) { +#if 0 + auto stream_data = nghttp2_session_get_stream_user_data(session, frame->hd.stream_id); + /* For DATA and HEADERS frame, this callback may be called after on_stream_close_callback. Check that stream still alive. */ + if (stream_data == nullptr) { + cerr<<"unable to find stream data!"<d_queries.at(frame->hd.stream_id); + BOOST_REQUIRE_GT(query.size(), sizeof(dnsheader)); + auto dh = reinterpret_cast(query.data()); + uint16_t id = ntohs(dh->id); + // cerr<<"got query ID "<(query.data()), query.size(), sizeof(dnsheader), false); + if (qname == DNSName("goaway.powerdns.com.")) { + conn->submitGoAway(); + } + else if (qname == DNSName("500.powerdns.com.") && (id % 2) == 0) { + /* we return a 500 on the first query only */ + conn->submitError(frame->hd.stream_id, 500, "Server failure"); + } + else if (qname == DNSName("wrong-stream-id.powerdns.com.") && (id % 2) == 0) { + /* we return a wrong stremad ID on the first query only */ + BOOST_CHECK_EQUAL(frame->hd.stream_id, 1); + conn->d_responses[frame->hd.stream_id] = expected.d_response; + /* use an invalid stream ID! */ + conn->d_idMapping[frame->hd.stream_id] = frame->hd.stream_id + 4; + conn->submitResponse(frame->hd.stream_id, conn->d_responses.at(frame->hd.stream_id)); + } + else { + conn->d_responses[frame->hd.stream_id] = expected.d_response; + conn->submitResponse(frame->hd.stream_id, conn->d_responses.at(frame->hd.stream_id)); + } + conn->d_queries.erase(frame->hd.stream_id); + } + + return 0; + } + + static int on_data_chunk_recv_callback(nghttp2_session* session, uint8_t flags, int32_t stream_id, const uint8_t* data, size_t len, void* user_data) + { + DOHConnection* conn = reinterpret_cast(user_data); + auto& query = conn->d_queries[stream_id]; + query.insert(query.end(), data, data + len); + return 0; + } + + static int on_stream_close_callback(nghttp2_session* session, int32_t stream_id, uint32_t error_code, void* user_data) + { + if (error_code == 0) { + return 0; + } + + return 0; + } +}; + +static std::map> s_connectionBuffers; + +class MockupTLSConnection : public TLSConnection +{ +public: + MockupTLSConnection(int descriptor, bool client = false, bool needProxyProtocol = false) : + d_descriptor(descriptor), d_client(client) + { + s_connectionBuffers[d_descriptor] = std::make_unique(needProxyProtocol); + } + + ~MockupTLSConnection() {} + + IOState tryHandshake() override + { + auto step = getStep(); + BOOST_REQUIRE_EQUAL(step.request, ExpectedStep::ExpectedRequest::handshakeClient); + + return step.nextState; + } + + IOState tryWrite(const PacketBuffer& buffer, size_t& pos, size_t toWrite) override + { + auto& conn = s_connectionBuffers.at(d_descriptor); + auto step = getStep(); + BOOST_REQUIRE_EQUAL(step.request, !d_client ? ExpectedStep::ExpectedRequest::writeToClient : ExpectedStep::ExpectedRequest::writeToBackend); + + if (step.bytes == 0) { + if (step.nextState == IOState::NeedWrite) { + return step.nextState; + } + throw std::runtime_error("Remote host closed the connection"); + } + + toWrite -= pos; + BOOST_REQUIRE_GE(buffer.size(), pos + toWrite); + + if (step.bytes < toWrite) { + toWrite = step.bytes; + } + + conn->submitIncoming(buffer, pos, toWrite); + pos += toWrite; + + return step.nextState; + } + + IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead, bool allowIncomplete = false) override + { + auto& conn = s_connectionBuffers.at(d_descriptor); + auto step = getStep(); + BOOST_REQUIRE_EQUAL(step.request, !d_client ? ExpectedStep::ExpectedRequest::readFromClient : ExpectedStep::ExpectedRequest::readFromBackend); + + if (step.bytes == 0) { + if (step.nextState == IOState::NeedRead) { + return step.nextState; + } + throw std::runtime_error("Remote host closed the connection"); + } + + auto& externalBuffer = conn->d_serverOutBuffer; + toRead -= pos; + + if (step.bytes < toRead) { + toRead = step.bytes; + } + if (allowIncomplete) { + if (toRead > externalBuffer.size()) { + toRead = externalBuffer.size(); + } + } + else { + BOOST_REQUIRE_GE(externalBuffer.size(), toRead); + } + + BOOST_REQUIRE_GE(buffer.size(), toRead); + + std::copy(externalBuffer.begin(), externalBuffer.begin() + toRead, buffer.begin() + pos); + pos += toRead; + externalBuffer.erase(externalBuffer.begin(), externalBuffer.begin() + toRead); + + return step.nextState; + } + + IOState tryConnect(bool fastOpen, const ComboAddress& remote) override + { + auto step = getStep(); + BOOST_REQUIRE_EQUAL(step.request, ExpectedStep::ExpectedRequest::connectToBackend); + + return step.nextState; + } + + void close() override + { + auto step = getStep(); + BOOST_REQUIRE_EQUAL(step.request, !d_client ? ExpectedStep::ExpectedRequest::closeClient : ExpectedStep::ExpectedRequest::closeBackend); + } + + bool hasBufferedData() const override + { + return false; + } + + bool isUsable() const override + { + return true; + } + + std::string getServerNameIndication() const override + { + return ""; + } + + std::vector getNextProtocol() const override + { + return std::vector(); + } + + LibsslTLSVersion getTLSVersion() const override + { + return LibsslTLSVersion::TLS13; + } + + bool hasSessionBeenResumed() const override + { + return false; + } + + std::vector> getSessions() override + { + return {}; + } + + void setSession(std::unique_ptr& session) override + { + } + + /* unused in that context, don't bother */ + void doHandshake() override + { + } + + void connect(bool fastOpen, const ComboAddress& remote, const struct timeval& timeout) override + { + } + + size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout = {0, 0}, bool allowIncomplete = false) override + { + return 0; + } + + size_t write(const void* buffer, size_t bufferSize, const struct timeval& writeTimeout) override + { + return 0; + } + +private: + ExpectedStep getStep() const + { + BOOST_REQUIRE(!s_steps.empty()); + auto step = s_steps.front(); + s_steps.pop_front(); + + if (step.cb) { + step.cb(d_descriptor); + } + + return step; + } + + const int d_descriptor; + bool d_client{false}; +}; + +class MockupTLSCtx : public TLSCtx +{ +public: + ~MockupTLSCtx() + { + } + + std::unique_ptr getConnection(int socket, const struct timeval& timeout, time_t now) override + { + return std::make_unique(socket); + } + + std::unique_ptr getClientConnection(const std::string& host, int socket, const struct timeval& timeout) override + { + return std::make_unique(socket, true, d_needProxyProtocol); + } + + void rotateTicketsKey(time_t now) override + { + } + + size_t getTicketsKeysCount() override + { + return 0; + } + + std::string getName() const override + { + return "Mockup TLS"; + } + + bool d_needProxyProtocol{false}; +}; + +class MockupFDMultiplexer : public FDMultiplexer +{ +public: + MockupFDMultiplexer() + { + } + + ~MockupFDMultiplexer() + { + } + + int run(struct timeval* tv, int timeout = 500) override + { + int ret = 0; + + gettimeofday(tv, nullptr); // MANDATORY + + /* 'ready' might be altered by a callback while we are iterating */ + const auto readyFDs = ready; + for (const auto fd : readyFDs) { + { + const auto& it = d_readCallbacks.find(fd); + + if (it != d_readCallbacks.end()) { + it->d_callback(it->d_fd, it->d_parameter); + } + } + + { + const auto& it = d_writeCallbacks.find(fd); + + if (it != d_writeCallbacks.end()) { + it->d_callback(it->d_fd, it->d_parameter); + } + } + } + + return ret; + } + + void getAvailableFDs(std::vector& fds, int timeout) override + { + } + + void addFD(int fd, FDMultiplexer::EventKind kind) override + { + } + + void removeFD(int fd, FDMultiplexer::EventKind) override + { + } + + string getName() const override + { + return "mockup"; + } + + void setReady(int fd) + { + ready.insert(fd); + } + + void setNotReady(int fd) + { + ready.erase(fd); + } + +private: + std::set ready; +}; + +class MockupQuerySender : public TCPQuerySender +{ +public: + bool active() const override + { + return true; + } + + const ClientState* getClientState() const override + { + return nullptr; + } + + void handleResponse(const struct timeval& now, TCPResponse&& response) override + { + if (d_customHandler) { + d_customHandler(d_id, now, std::move(response)); + return; + } + + BOOST_REQUIRE_GT(response.d_buffer.size(), sizeof(dnsheader)); + auto dh = reinterpret_cast(response.d_buffer.data()); + uint16_t id = ntohs(dh->id); + + BOOST_REQUIRE_EQUAL(id, d_id); + const auto& expected = s_responses.at(id); + BOOST_REQUIRE_EQUAL(expected.d_response.size(), response.d_buffer.size()); + for (size_t idx = 0; idx < response.d_buffer.size(); idx++) { + if (expected.d_response.at(idx) != response.d_buffer.at(idx)) { + cerr << "Mismatch at offset " << idx << ", expected " << std::to_string(response.d_buffer.at(idx)) << " got " << std::to_string(expected.d_response.at(idx)) << endl; + BOOST_CHECK(false); + } + } + + if (expected.d_response != response.d_buffer) { + BOOST_REQUIRE(false); + } + d_valid = true; + } + + void handleXFRResponse(const struct timeval& now, TCPResponse&& response) override + { + } + + void notifyIOError(IDState&& query, const struct timeval& now) override + { + d_error = true; + } + + std::function d_customHandler; + uint16_t d_id{0}; + bool d_valid{false}; + bool d_error{false}; +}; + +static bool isIPv6Supported() +{ + try { + ComboAddress addr("[2001:db8:53::1]:53"); + auto socket = std::make_unique(addr.sin4.sin_family, SOCK_STREAM, 0); + socket->setNonBlocking(); + int res = SConnectWithTimeout(socket->getHandle(), addr, timeval{0, 0}); + if (res == 0 || res == EINPROGRESS) { + return true; + } + return false; + } + catch (const std::exception& e) { + return false; + } +} + +static ComboAddress getBackendAddress(const std::string& lastDigit, uint16_t port) +{ + static const bool useV6 = isIPv6Supported(); + + if (useV6) { + return ComboAddress("2001:db8:53::" + lastDigit, port); + } + + return ComboAddress("192.0.2." + lastDigit, port); +} + +static std::unique_ptr s_mplexer; + +struct TestFixture +{ + TestFixture() + { + s_steps.clear(); + s_responses.clear(); + s_mplexer = std::make_unique(); + } + ~TestFixture() + { + clearH2Connections(); + s_steps.clear(); + s_responses.clear(); + s_mplexer.reset(); + } +}; + +BOOST_FIXTURE_TEST_CASE(test_SingleQuery, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + size_t counter = 1; + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and response from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as NOT ready anymore */ + dynamic_cast(s_mplexer.get())->setNotReady(desc); + }}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + s_connectionBuffers.at(desc)->submitGoAway(); + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + auto sliced = std::shared_ptr(sender); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(internalQuery), false); + BOOST_CHECK_EQUAL(result, true); + + while (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0) { + s_mplexer->run(&now); + } + BOOST_CHECK_EQUAL(sender->d_valid, true); +} + +BOOST_FIXTURE_TEST_CASE(test_ConcurrentQueries, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + s_connectionBuffers.at(desc)->submitGoAway(); + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0) { + s_mplexer->run(&now); + } + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, true); + } +} + +BOOST_FIXTURE_TEST_CASE(test_ConnectionReuse, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool firstQueryDone = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [&firstQueryDone](int desc) { + firstQueryDone = true; + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + }}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* later the backend sends a go away frame */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + s_connectionBuffers.at(desc)->submitGoAway(); + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + { + auto& query = queries.at(0); + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + + while (!firstQueryDone && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + BOOST_CHECK_EQUAL(query.first->d_valid, true); + BOOST_CHECK_EQUAL(firstQueryDone, true); + } + + { + auto& query = queries.at(1); + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + + while (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0) { + s_mplexer->run(&now); + } + + BOOST_CHECK_EQUAL(query.first->d_valid, true); + } +} + +BOOST_FIXTURE_TEST_CASE(test_InvalidDNSAnswer, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + size_t counter = 1; + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + /* TRUNCATE the answer */ + response.resize(11); + s_responses[counter] = {query, response}; + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + auto sender = std::make_shared(); + sender->d_id = counter; + sender->d_customHandler = [](uint16_t id, const struct timeval&, TCPResponse&& resp) { + BOOST_CHECK_EQUAL(resp.d_buffer.size(), 11U); + /* simulate an exception, since DoH and UDP frontends will process the query right away, + while TCP and DoT will first pass it back to the TCP worker thread */ + throw std::runtime_error("Invalid response"); + }; + InternalQuery internalQuery(std::move(query), IDState()); + + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and response from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* try to read, the backend says to go away */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + s_connectionBuffers.at(desc)->submitGoAway(); + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + auto sliced = std::shared_ptr(sender); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(internalQuery), false); + BOOST_CHECK_EQUAL(result, true); + + while (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0) { + s_mplexer->run(&now); + } + BOOST_CHECK_EQUAL(sender->d_valid, false); +} + +BOOST_FIXTURE_TEST_CASE(test_TimeoutWhileWriting, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool timeout = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::NeedWrite, std::numeric_limits::max(), [&timeout](int desc) { + timeout = true; + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (!timeout && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + struct timeval later = now; + later.tv_sec += backend->tcpSendTimeout + 1; + + auto expiredConns = handleH2Timeouts(*s_mplexer, later); + BOOST_CHECK_EQUAL(expiredConns, 1U); + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, false); + BOOST_CHECK_EQUAL(query.first->d_error, true); + } + + BOOST_CHECK_EQUAL(clearH2Connections(), 0U); +} + +BOOST_FIXTURE_TEST_CASE(test_TimeoutWhileReading, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool timeout = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [&timeout](int desc) { + /* set the timeout flag now, since the timeout occurs while waiting for the descriptor to become readable */ + timeout = true; + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (!timeout && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + struct timeval later = now; + later.tv_sec += backend->tcpRecvTimeout + 1; + + auto expiredConns = handleH2Timeouts(*s_mplexer, later); + BOOST_CHECK_EQUAL(expiredConns, 1U); + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, false); + BOOST_CHECK_EQUAL(query.first->d_error, true); + } + BOOST_CHECK_EQUAL(clearH2Connections(), 0U); +} + +BOOST_FIXTURE_TEST_CASE(test_ShortWrite, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool done = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::NeedWrite, 2, [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* settings (second attempt) + headers + data + headers (second query) + data */ + { + ExpectedStep::ExpectedRequest::writeToBackend, + IOState::Done, + std::numeric_limits::max(), + }, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [&done](int desc) { + /* mark backend as not ready */ + dynamic_cast(s_mplexer.get())->setNotReady(desc); + done = true; + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (!done && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, true); + } + + BOOST_CHECK_EQUAL(clearH2Connections(), 1U); +} + +BOOST_FIXTURE_TEST_CASE(test_ShortRead, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool done = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 4}, + /* read settings, headers and responses (second attempt) */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [&done](int desc) { + /* mark backend as not ready */ + dynamic_cast(s_mplexer.get())->setNotReady(desc); + done = true; + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (!done && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, true); + } + + BOOST_CHECK_EQUAL(clearH2Connections(), 1U); +} + +BOOST_FIXTURE_TEST_CASE(test_ConnectionClosedWhileReading, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 0}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0) { + s_mplexer->run(&now); + } + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, false); + BOOST_CHECK_EQUAL(query.first->d_error, true); + } + + BOOST_CHECK_EQUAL(clearH2Connections(), 0U); +} + +BOOST_FIXTURE_TEST_CASE(test_ConnectionClosedWhileWriting, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool done = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers, connection is closed by the backend */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, 0}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and response from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [&done](int desc) { + /* mark backend as not ready */ + dynamic_cast(s_mplexer.get())->setNotReady(desc); + done = true; + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (!done && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + BOOST_CHECK_EQUAL(queries.at(0).first->d_valid, false); + BOOST_CHECK_EQUAL(queries.at(0).first->d_error, true); + BOOST_CHECK_EQUAL(queries.at(1).first->d_valid, true); + BOOST_CHECK_EQUAL(queries.at(1).first->d_error, false); + + BOOST_CHECK_EQUAL(clearH2Connections(), 1U); +} + +BOOST_FIXTURE_TEST_CASE(test_GoAwayFromServer, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + /* set the number of reconnection attempts to a low value to not waste time */ + backend->d_retries = 1; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("goaway.powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read GO AWAY from the server (1) */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* close the first connection. It happens now because the new connection was set up first, then that one destroyed */ + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + /* read GO AWAY from the server (1) */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0) { + s_mplexer->run(&now); + } + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, false); + BOOST_CHECK_EQUAL(query.first->d_error, true); + } + + BOOST_CHECK_EQUAL(clearH2Connections(), 0U); +} + +BOOST_FIXTURE_TEST_CASE(test_HTTP500FromServer, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("500.powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool done = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [&done](int desc) { + /* mark backend as not ready */ + dynamic_cast(s_mplexer.get())->setNotReady(desc); + done = true; + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (!done && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + BOOST_CHECK_EQUAL(queries.at(0).first->d_valid, false); + BOOST_CHECK_EQUAL(queries.at(0).first->d_error, true); + BOOST_CHECK_EQUAL(queries.at(1).first->d_valid, true); + BOOST_CHECK_EQUAL(queries.at(1).first->d_error, false); + + BOOST_CHECK_EQUAL(clearH2Connections(), 1U); +} + +BOOST_FIXTURE_TEST_CASE(test_WrongStreamID, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("wrong-stream-id.powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + InternalQuery internalQuery(std::move(query), IDState()); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + bool timeout = false; + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* read ends up as a time out since nghttp2 filters the frame with the wrong stream ID */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&timeout](int desc) { + /* set the timeout flag now, since the timeout occurs while waiting for the descriptor to become readable */ + timeout = true; + }}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (!timeout && (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0)) { + s_mplexer->run(&now); + } + + struct timeval later = now; + later.tv_sec += backend->tcpRecvTimeout + 1; + + auto expiredConns = handleH2Timeouts(*s_mplexer, later); + BOOST_CHECK_EQUAL(expiredConns, 1U); + + BOOST_CHECK_EQUAL(queries.at(0).first->d_valid, false); + BOOST_CHECK_EQUAL(queries.at(0).first->d_error, true); + BOOST_CHECK_EQUAL(queries.at(1).first->d_valid, false); + BOOST_CHECK_EQUAL(queries.at(1).first->d_error, true); + + BOOST_CHECK_EQUAL(clearH2Connections(), 0U); +} + +BOOST_FIXTURE_TEST_CASE(test_ProxyProtocol, TestFixture) +{ + auto local = getBackendAddress("1", 80); + ClientState localCS(local, true, false, false, "", {}); + auto tlsCtx = std::make_shared(); + tlsCtx->d_needProxyProtocol = true; + localCS.tlsFrontend = std::make_shared(tlsCtx); + + struct timeval now; + gettimeofday(&now, nullptr); + + auto backend = std::make_shared(getBackendAddress("42", 53)); + backend->d_tlsCtx = tlsCtx; + backend->d_tlsSubjectName = "backend.powerdns.com"; + backend->d_dohPath = "/dns-query"; + backend->d_addXForwardedHeaders = true; + backend->useProxyProtocol = true; + + size_t numberOfQueries = 2; + std::vector, InternalQuery>> queries; + for (size_t counter = 0; counter < numberOfQueries; counter++) { + DNSName name("powerdns.com."); + PacketBuffer query; + GenericDNSPacketWriter pwQ(query, name, QType::A, QClass::IN, 0); + pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = htons(counter); + + PacketBuffer response; + GenericDNSPacketWriter pwR(response, name, QType::A, QClass::IN, 0); + pwR.getHeader()->qr = 1; + pwR.getHeader()->rd = 1; + pwR.getHeader()->ra = 1; + pwR.getHeader()->id = htons(counter); + pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); + pwR.xfr32BitInt(0x01020304); + pwR.commit(); + + s_responses[counter] = {query, response}; + + auto sender = std::make_shared(); + sender->d_id = counter; + std::string payload = makeProxyHeader(counter % 2, local, local, {}); + InternalQuery internalQuery(std::move(query), IDState()); + internalQuery.d_proxyProtocolPayload = std::move(payload); + queries.push_back({std::move(sender), std::move(internalQuery)}); + } + + s_steps = { + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* proxy protocol data + opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + {ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done}, + /* proxy protocol data + opening */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* headers */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + /* data */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max(), [](int desc) { + /* set the outgoing descriptor (backend connection) as ready */ + dynamic_cast(s_mplexer.get())->setReady(desc); + }}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + /* read settings, headers and responses from the server */ + {ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, std::numeric_limits::max()}, + /* acknowledge settings */ + {ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, std::numeric_limits::max()}, + {ExpectedStep::ExpectedRequest::closeBackend, IOState::Done}, + }; + + for (auto& query : queries) { + auto sliced = std::static_pointer_cast(query.first); + bool result = sendH2Query(backend, s_mplexer, sliced, std::move(query.second), false); + BOOST_CHECK_EQUAL(result, true); + } + + while (s_mplexer->getWatchedFDCount(false) != 0 || s_mplexer->getWatchedFDCount(true) != 0) { + s_mplexer->run(&now); + } + + for (auto& query : queries) { + BOOST_CHECK_EQUAL(query.first->d_valid, true); + } + + BOOST_CHECK_EQUAL(clearH2Connections(), 0U); +} + +BOOST_AUTO_TEST_SUITE_END(); +#endif /* HAVE_NGHTTP2 */ diff --git a/pdns/dnsdistdist/test-dnsdistrings_cc.cc b/pdns/dnsdistdist/test-dnsdistrings_cc.cc index b0194bc95d09..db665bb5fdf6 100644 --- a/pdns/dnsdistdist/test-dnsdistrings_cc.cc +++ b/pdns/dnsdistdist/test-dnsdistrings_cc.cc @@ -30,12 +30,14 @@ static void test_ring(size_t maxEntries, size_t numberOfShards, size_t nbLockTri ComboAddress requestor2("192.0.2.2"); uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; struct timespec now; gettime(&now); /* fill the query ring */ for (size_t idx = 0; idx < maxEntries; idx++) { - rings.insertQuery(now, requestor1, qname, qtype, size, dh); + rings.insertQuery(now, requestor1, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(rings.getNumberOfQueryEntries(), maxEntries); BOOST_CHECK_EQUAL(rings.getNumberOfResponseEntries(), 0U); @@ -53,7 +55,7 @@ static void test_ring(size_t maxEntries, size_t numberOfShards, size_t nbLockTri /* push enough queries to get rid of the existing ones */ for (size_t idx = 0; idx < maxEntries; idx++) { - rings.insertQuery(now, requestor2, qname, qtype, size, dh); + rings.insertQuery(now, requestor2, qname, qtype, size, dh, protocol); } BOOST_CHECK_EQUAL(rings.getNumberOfQueryEntries(), maxEntries); BOOST_CHECK_EQUAL(rings.getNumberOfResponseEntries(), 0U); @@ -74,7 +76,7 @@ static void test_ring(size_t maxEntries, size_t numberOfShards, size_t nbLockTri /* fill the response ring */ for (size_t idx = 0; idx < maxEntries; idx++) { - rings.insertResponse(now, requestor1, qname, qtype, latency, size, dh, server); + rings.insertResponse(now, requestor1, qname, qtype, latency, size, dh, server, outgoingProtocol); } BOOST_CHECK_EQUAL(rings.getNumberOfQueryEntries(), maxEntries); BOOST_CHECK_EQUAL(rings.getNumberOfResponseEntries(), maxEntries); @@ -94,7 +96,7 @@ static void test_ring(size_t maxEntries, size_t numberOfShards, size_t nbLockTri /* push enough responses to get rid of the existing ones */ for (size_t idx = 0; idx < maxEntries; idx++) { - rings.insertResponse(now, requestor2, qname, qtype, latency, size, dh, server); + rings.insertResponse(now, requestor2, qname, qtype, latency, size, dh, server, outgoingProtocol); } BOOST_CHECK_EQUAL(rings.getNumberOfQueryEntries(), maxEntries); BOOST_CHECK_EQUAL(rings.getNumberOfResponseEntries(), maxEntries); @@ -171,8 +173,8 @@ static void ringReaderThread(Rings& rings, std::atomic& done, size_t numbe static void ringWriterThread(Rings& rings, size_t numberOfEntries, const Rings::Query& query, const Rings::Response& response) { for (size_t idx = 0; idx < numberOfEntries; idx++) { - rings.insertQuery(query.when, query.requestor, query.name, query.qtype, query.size, query.dh); - rings.insertResponse(response.when, response.requestor, response.name, response.qtype, response.usec, response.size, response.dh, response.ds); + rings.insertQuery(query.when, query.requestor, query.name, query.qtype, query.size, query.dh, query.protocol); + rings.insertResponse(response.when, response.requestor, response.name, response.qtype, response.usec, response.size, response.dh, response.ds, response.protocol); } } @@ -199,10 +201,12 @@ BOOST_AUTO_TEST_CASE(test_Rings_Threaded) { unsigned int latency = 100; uint16_t qtype = QType::AAAA; uint16_t size = 42; + dnsdist::Protocol protocol = dnsdist::Protocol::DoUDP; + dnsdist::Protocol outgoingProtocol = dnsdist::Protocol::DoUDP; Rings rings(numberOfEntries, numberOfShards, lockAttempts, true); - Rings::Query query({requestor, qname, now, dh, size, qtype}); - Rings::Response response({requestor, server, qname, now, dh, latency, size, qtype}); + Rings::Query query({requestor, qname, now, dh, size, qtype, protocol}); + Rings::Response response({requestor, server, qname, now, dh, latency, size, qtype, outgoingProtocol}); std::atomic done(false); std::vector writerThreads; diff --git a/pdns/dnsdistdist/test-dnsdistrules_cc.cc b/pdns/dnsdistdist/test-dnsdistrules_cc.cc index 70ccbd0020e5..b44374fefab3 100644 --- a/pdns/dnsdistdist/test-dnsdistrules_cc.cc +++ b/pdns/dnsdistdist/test-dnsdistrules_cc.cc @@ -7,6 +7,23 @@ #include "dnsdist-rules.hh" +static DNSQuestion getDQ(const DNSName* providedName = nullptr) +{ + static const DNSName qname("powerdns.com."); + static const ComboAddress lc("127.0.0.1:53"); + static const ComboAddress rem("192.0.2.1:42"); + static struct timespec queryRealTime; + static PacketBuffer packet(sizeof(dnsheader)); + + uint16_t qtype = QType::A; + uint16_t qclass = QClass::IN; + auto proto = dnsdist::Protocol::DoUDP; + gettime(&queryRealTime, true); + + DNSQuestion dq(providedName ? providedName : &qname, qtype, qclass, &lc, &rem, packet, proto, &queryRealTime); + return dq; +} + BOOST_AUTO_TEST_SUITE(dnsdistluarules_cc) BOOST_AUTO_TEST_CASE(test_MaxQPSIPRule) { @@ -101,5 +118,32 @@ BOOST_AUTO_TEST_CASE(test_MaxQPSIPRule) { BOOST_CHECK_EQUAL(scanned, 0U); } +BOOST_AUTO_TEST_CASE(test_poolOutstandingRule) { + auto dq = getDQ(); + + ServerPool sp{}; + auto ds1 = std::make_shared(ComboAddress("192.0.2.1:53")); + auto ds2 = std::make_shared(ComboAddress("192.0.2.2:53")); + + /* increase the outstanding count of both */ + ds1->outstanding = 400; + ds2->outstanding = 30; + + sp.addServer(ds1); + sp.addServer(ds2); + + BOOST_CHECK_EQUAL(sp.poolLoad(), 400U + 30U); + + auto localPool = g_pools.getCopy(); + addServerToPool(localPool, "test", ds1); + addServerToPool(localPool, "test", ds2); + g_pools.setState(localPool); + + PoolOutstandingRule pOR1("test", 10); + BOOST_CHECK_EQUAL(pOR1.matches(&dq), true); + + PoolOutstandingRule pOR2("test", 1000); + BOOST_CHECK_EQUAL(pOR2.matches(&dq), false); +} BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/dnsdistdist/test-dnsdistsvc_cc.cc b/pdns/dnsdistdist/test-dnsdistsvc_cc.cc new file mode 100644 index 000000000000..c15d07a51caa --- /dev/null +++ b/pdns/dnsdistdist/test-dnsdistsvc_cc.cc @@ -0,0 +1,96 @@ + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_NO_MAIN + +#include + +#include "dnsdist-svc.hh" +#include "svc-records.hh" +#include "dnsparser.hh" + +BOOST_AUTO_TEST_SUITE(dnsdistsvc_cc) + +BOOST_AUTO_TEST_CASE(test_Basic) +{ + DNSName target("powerdns.com."); + + { + std::vector payload; + const uint16_t priority = 1; + BOOST_CHECK(generateSVCPayload(payload, priority, target, {SvcParam::SvcParamKey::port}, {"dot"}, false, 853, std::string(), {ComboAddress("192.0.2.1")}, {ComboAddress("2001:db8::1")}, {})); + /* 2 octet field for SvcPriority as an integer in network byte order */ + /* uncompressed, fully-qualified TargetName */ + /* list of SvcParams as: + - 2 octet field containing the SvcParamKey as an integer in network byte order + - 2 octet field containing the length of the SvcParamValue as an integer between 0 and 65535 in network byte order (but constrained by the RDATA and DNS message sizes) + - an octet string of this length whose contents are in a format determined by the SvcParamKey + SvcParamKeys SHALL appear in increasing numeric order + */ + size_t expectedSize = (/* priority */ 2) + target.wirelength() + (/* mandatory */ 2 + 2 + 2) + (/* alpns with 1-byte length field for each value */ 2 + 2 + 4) + (/* no-alpn-default is false */ 0) + (/* port */ 2 + 2 + 2) + (/* ech */ 0) + (/* v4 hints */ 2 + 2 + 9) + (/* v6 hints */ 2 + 2 + 11); + BOOST_CHECK_EQUAL(payload.size(), expectedSize); + + std::set params; + PacketReader pr(std::string_view(reinterpret_cast(payload.data()), payload.size()), 0); + BOOST_CHECK_EQUAL(pr.get16BitInt(), priority); + + /* we can't use getName() directly because it assumes that there has to be a dnsheader before the name */ + DNSName parsedTarget(reinterpret_cast(payload.data()), payload.size(), pr.getPosition(), false /* uncompress */, nullptr /* qtype */, nullptr /* qclass */, nullptr /* consumed */, 0); + pr.skip(parsedTarget.wirelength()); + BOOST_CHECK_EQUAL(target.toString(), parsedTarget.toString()); + + pr.xfrSvcParamKeyVals(params); + BOOST_REQUIRE_EQUAL(params.size(), 5U); + auto param = params.begin(); + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::mandatory); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::alpn); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::port); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::ipv4hint); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::ipv6hint); + } + + { + std::vector payload; + const uint16_t priority = 2; + const std::string ech("whatever"); + const std::string dohParam("/dns-query{?dns}"); + + BOOST_CHECK(generateSVCPayload(payload, priority, target, {SvcParam::SvcParamKey::port}, {"h2"}, true, 443, ech, {ComboAddress("192.0.2.2")}, {ComboAddress("2001:db8::2")}, {std::pair(42, dohParam)})); + + size_t expectedSize = (/* priority */ 2) + target.wirelength() + (/* mandatory */ 2 + 2 + 2) + (/* alpns */ 2 + 2 + 3) + (/* no-alpn-default is true */ 2 + 2) + (/* port */ 2 + 2 + 2) + (/* ech */ 2 + 2 + ech.size()) + (/* v4 hints */ 2 + 2 + 9) + (/* v6 hints */ 2 + 2 + 11) + (/* doh parameter */ 2 + 2 + dohParam.size()); + BOOST_CHECK_EQUAL(payload.size(), expectedSize); + + std::set params; + PacketReader pr(std::string_view(reinterpret_cast(payload.data()), payload.size()), 0); + BOOST_CHECK_EQUAL(pr.get16BitInt(), priority); + + /* we can't use getName() directly because it assumes that there has to be a dnsheader before the name */ + DNSName parsedTarget(reinterpret_cast(payload.data()), payload.size(), pr.getPosition(), false /* uncompress */, nullptr /* qtype */, nullptr /* qclass */, nullptr /* consumed */, 0); + pr.skip(parsedTarget.wirelength()); + BOOST_CHECK_EQUAL(target.toString(), parsedTarget.toString()); + + pr.xfrSvcParamKeyVals(params); + BOOST_REQUIRE_EQUAL(params.size(), 8U); + auto param = params.begin(); + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::mandatory); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::alpn); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::no_default_alpn); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::port); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::ipv4hint); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::ech); + ++param; + BOOST_CHECK(param->getKey() == SvcParam::SvcParamKey::ipv6hint); + ++param; + BOOST_CHECK_EQUAL(static_cast(param->getKey()), 42U); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/dnsdistdist/test-dnsdisttcp_cc.cc b/pdns/dnsdistdist/test-dnsdisttcp_cc.cc index 516c2306e9d9..1fbb00e9ceb7 100644 --- a/pdns/dnsdistdist/test-dnsdisttcp_cc.cc +++ b/pdns/dnsdistdist/test-dnsdisttcp_cc.cc @@ -41,7 +41,9 @@ GlobalStateHolder g_dstates; QueryCount g_qcount; -bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::shared_ptr& dnsCryptQuery, time_t now, bool tcp) +const bool TCPIOHandler::s_disableConnectForUnitTests = true; + +bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::unique_ptr& dnsCryptQuery, time_t now, bool tcp) { return false; } @@ -61,7 +63,7 @@ uint64_t getLatencyCount(const std::string&) return 0; } -void handleResponseSent(const IDState& ids, double udiff, const ComboAddress& client, const ComboAddress& backend, unsigned int size, const dnsheader& cleartextDH) +void handleResponseSent(const IDState& ids, double udiff, const ComboAddress& client, const ComboAddress& backend, unsigned int size, const dnsheader& cleartextDH, dnsdist::Protocol protocol) { } @@ -105,11 +107,11 @@ struct ExpectedStep public: enum class ExpectedRequest { handshakeClient, readFromClient, writeToClient, closeClient, connectToBackend, readFromBackend, writeToBackend, closeBackend }; - ExpectedStep(ExpectedRequest r, IOState n, size_t b = 0, std::function fn = nullptr): cb(fn), request(r), nextState(n), bytes(b) + ExpectedStep(ExpectedRequest r, IOState n, size_t b = 0, std::function fn = nullptr): cb(fn), request(r), nextState(n), bytes(b) { } - std::function cb{nullptr}; + std::function cb{nullptr}; ExpectedRequest request; IOState nextState; size_t bytes{0}; @@ -174,7 +176,7 @@ class MockupTLSConnection : public TLSConnection return step.nextState; } - IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead) override + IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead, bool allowIncomplete=false) override { auto step = getStep(); BOOST_REQUIRE_EQUAL(step.request, !d_client ? ExpectedStep::ExpectedRequest::readFromClient : ExpectedStep::ExpectedRequest::readFromBackend); @@ -222,11 +224,21 @@ class MockupTLSConnection : public TLSConnection return false; } + bool isUsable() const override + { + return true; + } + std::string getServerNameIndication() const override { return ""; } + std::vector getNextProtocol() const override + { + return std::vector(); + } + LibsslTLSVersion getTLSVersion() const override { return LibsslTLSVersion::TLS13; @@ -237,9 +249,9 @@ class MockupTLSConnection : public TLSConnection return false; } - std::unique_ptr getSession() override + std::vector> getSessions() override { - return nullptr; + return {}; } void setSession(std::unique_ptr& session) override @@ -255,7 +267,7 @@ class MockupTLSConnection : public TLSConnection { } - size_t read(void* buffer, size_t bufferSize, const struct timeval&readTimeout, const struct timeval& totalTimeout={0,0}) override + size_t read(void* buffer, size_t bufferSize, const struct timeval&readTimeout, const struct timeval& totalTimeout={0,0}, bool allowIncomplete=false) override { return 0; } @@ -272,7 +284,7 @@ class MockupTLSConnection : public TLSConnection s_steps.pop_front(); if (step.cb) { - step.cb(d_descriptor, step); + step.cb(d_descriptor); } return step; @@ -386,9 +398,57 @@ class MockupFDMultiplexer : public FDMultiplexer std::set ready; }; +static bool isIPv6Supported() +{ + try { + ComboAddress addr("[2001:db8:53::1]:53"); + auto socket = std::make_unique(addr.sin4.sin_family, SOCK_STREAM, 0); + socket->setNonBlocking(); + int res = SConnectWithTimeout(socket->getHandle(), addr, timeval{0, 0}); + if (res == 0 || res == EINPROGRESS) { + return true; + } + return false; + } + catch (const std::exception& e) { + return false; + } +} + +static ComboAddress getBackendAddress(const std::string& lastDigit, uint16_t port) +{ + static const bool useV6 = isIPv6Supported(); + + if (useV6) { + return ComboAddress("2001:db8:53::" + lastDigit, port); + } + + return ComboAddress("192.0.2." + lastDigit, port); +} + +static void appendPayloadEditingID(PacketBuffer& buffer, const PacketBuffer& payload, uint16_t newID) +{ + PacketBuffer newPayload(payload); + dnsheader dh; + memcpy(&dh, &newPayload.at(sizeof(uint16_t)), sizeof(dh)); + dh.id = htons(newID); + memcpy(&newPayload.at(sizeof(uint16_t)), &dh, sizeof(dh)); + buffer.insert(buffer.end(), newPayload.begin(), newPayload.end()); +} + +static void prependPayloadEditingID(PacketBuffer& buffer, const PacketBuffer& payload, uint16_t newID) +{ + PacketBuffer newPayload(payload); + dnsheader dh; + memcpy(&dh, &newPayload.at(sizeof(uint16_t)), sizeof(dh)); + dh.id = htons(newID); + memcpy(&newPayload.at(sizeof(uint16_t)), &dh, sizeof(dh)); + buffer.insert(buffer.begin(), newPayload.begin(), newPayload.end()); +} + static void testInit(const std::string& name, TCPClientThreadData& threadData) { -#if 0 +#ifdef DEBUGLOG_ENABLED cerr<(); } @@ -410,7 +471,7 @@ static void testInit(const std::string& name, TCPClientThreadData& threadData) BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) { - ComboAddress local("192.0.2.1:80"); + auto local = getBackendAddress("1", 80); ClientState localCS(local, true, false, false, "", {}); auto tlsCtx = std::make_shared(); localCS.tlsFrontend = std::make_shared(tlsCtx); @@ -444,7 +505,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) return ProcessQueryResult::Drop; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); } @@ -467,7 +528,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) return ProcessQueryResult::SendAnswer; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size()); BOOST_CHECK(s_writeBuffer == query); @@ -502,7 +563,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) /* mark the incoming FD as always ready */ dynamic_cast(threadData.mplexer.get())->setReady(-1); - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { threadData.mplexer->run(&now); @@ -526,7 +587,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) throw std::runtime_error("Something unexpected happened"); }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); } @@ -554,7 +615,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) return ProcessQueryResult::SendAnswer; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size() * count); #endif @@ -580,7 +641,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) /* mark the incoming FD as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(-1); - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(threadData.mplexer->run(&now), 0); struct timeval later = now; @@ -616,7 +677,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) /* mark the incoming FD as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(-1); - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(threadData.mplexer->run(&now), 0); struct timeval later = now; @@ -649,7 +710,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) return ProcessQueryResult::SendAnswer; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); } @@ -657,7 +718,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_SelfAnswered) BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) { - ComboAddress local("192.0.2.1:80"); + auto local = getBackendAddress("1", 80); ClientState localCS(local, true, false, false, "", {}); auto tlsCtx = std::make_shared(); localCS.tlsFrontend = std::make_shared(tlsCtx); @@ -680,6 +741,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) TEST_INIT("=> reading PP"); g_proxyProtocolACL.addMask("0.0.0.0/0"); + g_proxyProtocolACL.addMask("::0/0"); auto proxyPayload = makeProxyHeader(true, ComboAddress("192.0.2.1"), ComboAddress("192.0.2.2"), {}); BOOST_REQUIRE_GT(proxyPayload.size(), s_proxyProtocolMinimumHeaderSize); @@ -709,7 +771,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) /* mark the incoming FD as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(-1); - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(threadData.mplexer->run(&now), 0); BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size() * 2U); @@ -719,6 +781,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) TEST_INIT("=> Invalid PP"); g_proxyProtocolACL.addMask("0.0.0.0/0"); + g_proxyProtocolACL.addMask("::0/0"); auto proxyPayload = std::vector(s_proxyProtocolMinimumHeaderSize); std::fill(proxyPayload.begin(), proxyPayload.end(), 0); @@ -735,7 +798,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) return ProcessQueryResult::SendAnswer; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); @@ -745,13 +808,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) TEST_INIT("=> timeout while reading PP"); g_proxyProtocolACL.addMask("0.0.0.0/0"); + g_proxyProtocolACL.addMask("::0/0"); auto proxyPayload = makeProxyHeader(true, ComboAddress("192.0.2.1"), ComboAddress("192.0.2.2"), {}); BOOST_REQUIRE_GT(proxyPayload.size(), s_proxyProtocolMinimumHeaderSize); s_readBuffer = query; // preprend the proxy protocol payload s_readBuffer.insert(s_readBuffer.begin(), proxyPayload.begin(), proxyPayload.end()); - // append a second query - s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); s_steps = { { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, @@ -766,7 +828,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) /* mark the incoming FD as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(-1); - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(threadData.mplexer->run(&now), 0); struct timeval later = now; @@ -786,7 +848,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionWithProxyProtocol_SelfAnswered) BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) { - ComboAddress local("192.0.2.1:80"); + auto local = getBackendAddress("1", 80); ClientState localCS(local, true, false, false, "", {}); auto tlsCtx = std::make_shared(); localCS.tlsFrontend = std::make_shared(tlsCtx); @@ -800,6 +862,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) PacketBuffer query; GenericDNSPacketWriter pwQ(query, DNSName("powerdns.com."), QType::A, QClass::IN, 0); pwQ.getHeader()->rd = 1; + pwQ.getHeader()->id = 0; auto shortQuery = query; shortQuery.resize(sizeof(dnsheader) - 1); @@ -811,7 +874,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) const uint8_t sizeBytes[] = { static_cast(querySize / 256), static_cast(querySize % 256) }; query.insert(query.begin(), sizeBytes, sizeBytes + 2); - auto backend = std::make_shared(ComboAddress("192.0.2.42:53"), ComboAddress("0.0.0.0:0"), 0, std::string(), 1, false); + auto backend = std::make_shared(getBackendAddress("42", 53)); backend->d_tlsCtx = tlsCtx; { @@ -845,12 +908,13 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size()); BOOST_CHECK(s_writeBuffer == query); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); BOOST_CHECK(s_backendWriteBuffer == query); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -884,11 +948,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) throw std::runtime_error("Unexpected error while processing the response"); }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); BOOST_CHECK(s_backendWriteBuffer == query); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -922,11 +987,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return false; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); BOOST_CHECK(s_backendWriteBuffer == query); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -964,11 +1030,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); BOOST_CHECK(s_backendWriteBuffer == query); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -990,10 +1057,11 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), 0U); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -1027,10 +1095,11 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -1041,18 +1110,18 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) TEST_INIT("=> Short read and write to backend"); s_readBuffer = query; // append a second query - s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); + appendPayloadEditingID(s_readBuffer, query, 1); s_backendReadBuffer = query; // append a second query - s_backendReadBuffer.insert(s_backendReadBuffer.end(), query.begin(), query.end()); + appendPayloadEditingID(s_backendReadBuffer, query, 1); s_steps = { { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, query.size() - 2 }, /* connect to backend */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::NeedWrite, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::NeedWrite, 0, [&threadData](int desc) { /* set the outgoing descriptor (backend connection) as ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } @@ -1096,13 +1165,15 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) /* set the incoming descriptor as ready! */ dynamic_cast(threadData.mplexer.get())->setReady(-1); - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { threadData.mplexer->run(&now); } BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size() * 2U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size() * 2U); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1117,27 +1188,27 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, query.size() - 2 }, /* opening a connection to the backend (5 tries by default) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, @@ -1155,10 +1226,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), 0U); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1189,7 +1262,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); struct timeval later = now; later.tv_sec += backend->tcpSendTimeout + 1; @@ -1203,6 +1276,8 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) } BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), 0U); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1233,7 +1308,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); struct timeval later = now; later.tv_sec += backend->tcpRecvTimeout + 1; @@ -1247,6 +1322,8 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) } BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1288,10 +1365,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), 0U); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1342,11 +1421,13 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size()); BOOST_CHECK(s_writeBuffer == query); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1364,25 +1445,25 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, 0 }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* and now reconnection fails (1) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* 2 */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* 3 */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* 4 */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int descriptor) { throw NetworkError("Connection refused by the backend"); } }, @@ -1399,10 +1480,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), 0U); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1449,10 +1532,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); - BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size() * backend->retries); + BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size() * backend->d_retries); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1507,11 +1592,13 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size()); BOOST_CHECK(s_writeBuffer == query); - BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size() * backend->retries); + BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size() * backend->d_retries); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); } @@ -1546,11 +1633,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), query.size()); BOOST_CHECK(s_backendWriteBuffer == query); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -1568,8 +1656,8 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) s_readBuffer = query; for (size_t idx = 0; idx < count; idx++) { - s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), query.begin(), query.end()); + appendPayloadEditingID(s_readBuffer, query, idx); + appendPayloadEditingID(s_backendReadBuffer, query, idx); } s_steps = { { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, @@ -1607,9 +1695,10 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); BOOST_CHECK_EQUAL(s_writeBuffer.size(), query.size() * count); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -1622,7 +1711,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnection_BackendNoOOOR) BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { - ComboAddress local("192.0.2.1:80"); + auto local = getBackendAddress("1", 80); ClientState localCS(local, true, false, false, "", {}); /* enable out-of-order on the front side */ localCS.d_maxInFlightQueriesPerConn = 65536; @@ -1630,7 +1719,10 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) auto tlsCtx = std::make_shared(); localCS.tlsFrontend = std::make_shared(tlsCtx); - auto backend = std::make_shared(ComboAddress("192.0.2.42:53"), ComboAddress("0.0.0.0:0"), 0, std::string(), 1, false); + ConnectionInfo connInfo(&localCS); + connInfo.remote = getBackendAddress("84", 4242); + + auto backend = std::make_shared(getBackendAddress("42", 53)); backend->d_tlsCtx = tlsCtx; /* enable out-of-order on the backend side as well */ backend->d_maxInFlightQueriesPerConn = 65536; @@ -1651,7 +1743,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) for (auto& query : queries) { GenericDNSPacketWriter pwQ(query, DNSName("powerdns" + std::to_string(counter) + ".com."), QType::A, QClass::IN, 0); pwQ.getHeader()->rd = 1; - pwQ.getHeader()->id = counter; + pwQ.getHeader()->id = htons(counter); uint16_t querySize = static_cast(query.size()); const uint8_t sizeBytes[] = { static_cast(querySize / 256), static_cast(querySize % 256) }; query.insert(query.begin(), sizeBytes, sizeBytes + 2); @@ -1667,7 +1759,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) pwR.getHeader()->qr = 1; pwR.getHeader()->rd = 1; pwR.getHeader()->ra = 1; - pwR.getHeader()->id = counter; + pwR.getHeader()->id = htons(counter); pwR.startRecord(name, QType::A, 7200, QClass::IN, DNSResourceRecord::ANSWER); pwR.xfr32BitInt(0x01020304); pwR.commit(); @@ -1684,16 +1776,18 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) PacketBuffer expectedWriteBuffer; PacketBuffer expectedBackendWriteBuffer; + uint16_t backendCounter = 0; for (const auto& query : queries) { s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); + appendPayloadEditingID(expectedBackendWriteBuffer, query, backendCounter++); } - expectedBackendWriteBuffer = s_readBuffer; + backendCounter = 0; for (const auto& response : responses) { /* reverse order */ - s_backendReadBuffer.insert(s_backendReadBuffer.begin(), response.begin(), response.end()); + prependPayloadEditingID(s_backendReadBuffer, response, backendCounter++); + expectedWriteBuffer.insert(expectedWriteBuffer.begin(), response.begin(), response.end()); } - expectedWriteBuffer = s_backendReadBuffer; s_steps = { { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, @@ -1733,7 +1827,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* sending query to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(4).size() }, /* no response ready yet, but the backend becomes ready */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { /* set the outgoing descriptor (backend connection) as ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -1769,7 +1863,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() - 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() }, /* sending it to the client, the client descriptor becomes ready */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc) { /* set the incoming descriptor (client connection) as ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -1790,7 +1884,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { threadData.mplexer->run(&now); @@ -1800,6 +1894,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), totalQueriesSize); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -1818,8 +1913,14 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); } - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(0).begin(), responses.at(0).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(4).begin(), responses.at(4).end()); + uint16_t backendCounter = 0; + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(2), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(4), backendCounter++); + + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), 0); + appendPayloadEditingID(s_backendReadBuffer, responses.at(4), 3); /* self-answered */ expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(3).begin(), responses.at(3).end()); @@ -1827,12 +1928,6 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(0).begin(), responses.at(0).end()); expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(4).begin(), responses.at(4).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(0).begin(), queries.at(0).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(1).begin(), queries.at(1).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(2).begin(), queries.at(2).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(4).begin(), queries.at(4).end()); - - bool timeout = false; s_steps = { { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, @@ -1871,12 +1966,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(4).size() }, /* reading a response from the backend (1) */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() - 2 }, - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size(), [&threadData](int desc) { /* set the backend descriptor as ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, /* sending it to the client (1) */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc) { /* set the client descriptor as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, @@ -1889,7 +1984,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size() }, /* try to read from the backend but there is no answer ready yet */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData, &timeout](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData, &timeout](int desc) { /* set the backend descriptor as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); timeout = true; @@ -1921,7 +2016,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { @@ -1943,6 +2038,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -1960,13 +2056,24 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) for (const auto& query : queries) { s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); } - expectedBackendWriteBuffer = s_readBuffer; - for (const auto& response : responses) { expectedWriteBuffer.insert(expectedWriteBuffer.end(), response.begin(), response.end()); } - s_backendReadBuffer = expectedWriteBuffer; + uint16_t backendCounter = 0; + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), backendCounter); + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), backendCounter); + appendPayloadEditingID(s_backendReadBuffer, responses.at(1), backendCounter++); + + // new connection + backendCounter = 0; + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(2), backendCounter); + appendPayloadEditingID(s_backendReadBuffer, responses.at(2), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(3), backendCounter); + appendPayloadEditingID(s_backendReadBuffer, responses.at(3), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(4), backendCounter); + appendPayloadEditingID(s_backendReadBuffer, responses.at(4), backendCounter++); bool timeout = false; int backendDesc; @@ -1981,7 +2088,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(0).size() }, /* read response size and the beginning of the response (1) from the backend */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 1, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 1, [&threadData](int desc) { /* set the backend descriptor as ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -1989,7 +2096,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(1).size() - 2 }, /* trying to read an additional query, if any */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc) { /* set the client descriptor as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, @@ -2003,7 +2110,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(1).size() - 2 }, /* sending response (2) to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(1).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(1).size(), [&threadData](int desc) { /* set the client descriptor as ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -2016,7 +2123,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* opening a connection to the backend */ { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done }, /* sending query (3) to the backend, short write */ - { ExpectedStep::ExpectedRequest::writeToBackend, IOState::NeedWrite, 1, [&threadData,&backendDesc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToBackend, IOState::NeedWrite, 1, [&threadData,&backendDesc](int desc) { /* set the backend descriptor as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); backendDesc = desc; @@ -2028,12 +2135,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(3).size() - 2 }, /* reading a query from the client (5) */ { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, - { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(4).size() - 2, [&threadData,&backendDesc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(4).size() - 2, [&threadData,&backendDesc](int desc) { /* set the backend descriptor as ready now */ dynamic_cast(threadData.mplexer.get())->setReady(backendDesc); } }, /* nothing else to read from the client for now */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc) { /* set the client descriptor as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, @@ -2052,13 +2159,13 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(3).size() - 2 }, /* sending it to the client (4) but short write */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::NeedWrite, responses.at(3).size() - 1, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::NeedWrite, responses.at(3).size() - 1, [&threadData](int desc) { /* set the client descriptor as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, /* reading a response from the backend (5) */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(4).size() - 2, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(4).size() - 2, [&threadData](int desc) { /* set the client descriptor as ready to resume sending */ dynamic_cast(threadData.mplexer.get())->setReady(-1); } }, @@ -2068,7 +2175,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size() }, /* nothing to read from the client, then timeout later */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData,&timeout](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData,&timeout](int desc) { /* set the client descriptor as NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); timeout = true; @@ -2089,7 +2196,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { @@ -2111,6 +2218,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -2144,7 +2252,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(1).size() - 2 }, /* query is dropped, closing the connection to the client */ - { ExpectedStep::ExpectedRequest::closeClient, IOState::Done, 0, [&timeout](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::closeClient, IOState::Done, 0, [&timeout](int desc) { timeout = true; } }, /* closing a connection to the backend after a timeout */ @@ -2164,7 +2272,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { threadData.mplexer->run(&now); @@ -2185,6 +2293,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -2211,7 +2320,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(0).size() - 2 }, /* opening a connection to the backend */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptor](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptor](int desc) { backendDescriptor = desc; } }, /* sending query to the backend */ @@ -2222,7 +2331,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(1).size() - 2 }, /* query is dropped, closing the connection to the client */ - { ExpectedStep::ExpectedRequest::closeClient, IOState::Done, 0, [&threadData,&backendDescriptor](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::closeClient, IOState::Done, 0, [&threadData,&backendDescriptor](int desc) { /* the backend descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(backendDescriptor); } }, @@ -2246,7 +2355,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while ((threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { threadData.mplexer->run(&now); @@ -2256,6 +2365,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -2271,15 +2381,18 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) s_readBuffer.insert(s_readBuffer.end(), queries.at(1).begin(), queries.at(1).end()); s_readBuffer.insert(s_readBuffer.end(), queries.at(4).begin(), queries.at(4).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(0).begin(), queries.at(0).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(1).begin(), queries.at(1).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(4).begin(), queries.at(4).end()); + uint16_t backendCounter = 0; + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(4), backendCounter++); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(1).begin(), responses.at(1).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(0).begin(), responses.at(0).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(4).begin(), responses.at(4).end()); + appendPayloadEditingID(s_backendReadBuffer, responses.at(1), 1); + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), 0); + appendPayloadEditingID(s_backendReadBuffer, responses.at(4), 2); - expectedWriteBuffer = s_backendReadBuffer; + appendPayloadEditingID(expectedWriteBuffer, responses.at(1), 1); + appendPayloadEditingID(expectedWriteBuffer, responses.at(0), 0); + appendPayloadEditingID(expectedWriteBuffer, responses.at(4), 4); /* make sure that the backend's timeout is longer than the client's */ backend->tcpRecvTimeout = 30; @@ -2303,7 +2416,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* sending query (2) to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(1).size() }, /* no response ready yet */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&timeout,&backendDescriptor](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&timeout,&backendDescriptor](int desc) { backendDescriptor = desc; timeout = true; } }, @@ -2318,7 +2431,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() - 2 }, /* sending response (1) to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc) { /* setting the client descriptor ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -2327,12 +2440,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(4).size() - 2 }, /* sending query (3) to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(4).size() }, - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { /* setting the backend descriptor NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, /* try to read from the client again, nothing yet */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData,&backendDescriptor](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData,&backendDescriptor](int desc) { /* the client descriptor becomes NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); /* the backend one is ready, though */ @@ -2342,7 +2455,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(4).size() - 2 }, /* sending response (3) to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size(), [&timeout](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size(), [&timeout](int desc) { timeout = true; } }, /* client times out again, this time we close the connection */ @@ -2359,7 +2472,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { threadData.mplexer->run(&now); @@ -2396,6 +2509,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -2444,12 +2558,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* sending query (3) to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(2).size() }, /* no response ready yet but the backend descriptor becomes ready */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { /* the backend descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, /* nothing from the client either */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc) { /* the client descriptor is NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, @@ -2466,31 +2580,31 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* so we close the connection */ { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* try opening a new connection to the backend, it fails (5) times */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [](int desc) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* try opening a new connection to the backend, it fails (5) times */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* try opening a new connection to the backend, it fails (5) times */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* try opening a new connection to the backend, it fails (5) times */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc) { throw NetworkError("Connection refused by the backend"); } }, { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, /* try opening a new connection to the backend, it fails (5) times */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done,0, [](int desc) { throw NetworkError("Connection refused by the backend"); } }, /* closing a connection to the backend, client becomes ready */ - { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done, 0, [&threadData](int desc) { /* the client descriptor is ready */ dynamic_cast(threadData.mplexer.get())->setReady(-1); } }, @@ -2510,7 +2624,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { threadData.mplexer->run(&now); @@ -2520,6 +2634,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -2641,14 +2756,16 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) s_readBuffer = axfrQuery; s_readBuffer.insert(s_readBuffer.end(), secondQuery.begin(), secondQuery.end()); - expectedBackendWriteBuffer = s_readBuffer; + uint16_t backendCounter = 0; + appendPayloadEditingID(expectedBackendWriteBuffer, axfrQuery, backendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, secondQuery, backendCounter++); for (const auto& response : axfrResponses) { - s_backendReadBuffer.insert(s_backendReadBuffer.end(), response.begin(), response.end()); + appendPayloadEditingID(s_backendReadBuffer, response, 0); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), response.begin(), response.end()); } - s_backendReadBuffer.insert(s_backendReadBuffer.end(), secondResponse.begin(), secondResponse.end()); - - expectedWriteBuffer = s_backendReadBuffer; + appendPayloadEditingID(s_backendReadBuffer, secondResponse, 1); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), secondResponse.begin(), secondResponse.end()); bool timeout = false; s_steps = { @@ -2661,12 +2778,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* sending query (1) to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, axfrQuery.size() }, /* no response ready yet, but setting the backend descriptor readable */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { /* the backend descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, /* no more query from the client for now */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0 , [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0 , [&threadData](int desc) { /* the client descriptor becomes NOT ready */ dynamic_cast(threadData.mplexer.get())->setNotReady(-1); } }, @@ -2684,7 +2801,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, axfrResponses.at(2).size() - 2 }, /* sending response (3) to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, axfrResponses.at(2).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, axfrResponses.at(2).size(), [&threadData](int desc) { /* the client descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(-1); } }, @@ -2714,7 +2831,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { threadData.mplexer->run(&now); @@ -2901,15 +3018,18 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) s_readBuffer.insert(s_readBuffer.end(), ixfrQuery.begin(), ixfrQuery.end()); s_readBuffer.insert(s_readBuffer.end(), secondQuery.begin(), secondQuery.end()); - expectedBackendWriteBuffer = s_readBuffer; + appendPayloadEditingID(expectedBackendWriteBuffer, firstQuery, 0); + appendPayloadEditingID(expectedBackendWriteBuffer, ixfrQuery, 1); + appendPayloadEditingID(expectedBackendWriteBuffer, secondQuery, 2); - s_backendReadBuffer = firstResponse; + appendPayloadEditingID(s_backendReadBuffer, firstResponse, 0); + expectedWriteBuffer.insert(expectedWriteBuffer.begin(), firstResponse.begin(), firstResponse.end()); for (const auto& response : ixfrResponses) { - s_backendReadBuffer.insert(s_backendReadBuffer.end(), response.begin(), response.end()); + appendPayloadEditingID(s_backendReadBuffer, response, 1); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), response.begin(), response.end()); } - s_backendReadBuffer.insert(s_backendReadBuffer.end(), secondResponse.begin(), secondResponse.end()); - - expectedWriteBuffer = s_backendReadBuffer; + appendPayloadEditingID(s_backendReadBuffer, secondResponse, 2); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), secondResponse.begin(), secondResponse.end()); bool timeout = false; s_steps = { @@ -2922,7 +3042,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* sending query (1) to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, firstQuery.size() }, /* no response ready yet, but setting the backend descriptor readable */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { /* the backend descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -2932,7 +3052,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, firstResponse.size() - 2 }, /* sending response (1) to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, firstResponse.size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, firstResponse.size(), [&threadData](int desc) { /* client descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(-1); } }, @@ -2945,7 +3065,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, ixfrResponses.at(0).size() - 2 }, /* sending response (ixfr 1) to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, ixfrResponses.at(0).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, ixfrResponses.at(0).size(), [&threadData](int desc) { /* the client descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(-1); } }, @@ -2975,7 +3095,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { threadData.mplexer->run(&now); @@ -2985,6 +3105,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ IncomingTCPConnectionState::clearAllDownstreamConnections(); @@ -2996,33 +3117,36 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) PacketBuffer expectedWriteBuffer; PacketBuffer expectedBackendWriteBuffer; - auto proxyPayload = makeProxyHeader(true, ComboAddress("0.0.0.0"), local, {}); + auto proxyPayload = makeProxyHeader(true, getBackendAddress("84", 4242), local, {}); BOOST_REQUIRE_GT(proxyPayload.size(), s_proxyProtocolMinimumHeaderSize); s_readBuffer.insert(s_readBuffer.end(), queries.at(0).begin(), queries.at(0).end()); s_readBuffer.insert(s_readBuffer.end(), queries.at(1).begin(), queries.at(1).end()); s_readBuffer.insert(s_readBuffer.end(), queries.at(2).begin(), queries.at(2).end()); - auto proxyEnabledBackend = std::make_shared(ComboAddress("192.0.2.42:53"), ComboAddress("0.0.0.0:0"), 0, std::string(), 1, false); + auto proxyEnabledBackend = std::make_shared(getBackendAddress("42", 53)); proxyEnabledBackend->d_tlsCtx = tlsCtx; /* enable out-of-order on the backend side as well */ proxyEnabledBackend->d_maxInFlightQueriesPerConn = 65536; - proxyEnabledBackend-> useProxyProtocol = true; + proxyEnabledBackend->useProxyProtocol = true; expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), proxyPayload.begin(), proxyPayload.end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(0).begin(), queries.at(0).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(1).begin(), queries.at(1).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(2).begin(), queries.at(2).end()); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), 1); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(2), 2); expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), proxyPayload.begin(), proxyPayload.end()); /* we are using an unordered_map, so all bets are off here :-/ */ - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(2).begin(), queries.at(2).end()); - expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), queries.at(1).begin(), queries.at(1).end()); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(2), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), 1); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(0).begin(), responses.at(0).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(1).begin(), responses.at(1).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(2).begin(), responses.at(2).end()); + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), 0); + /* after the reconnection */ + appendPayloadEditingID(s_backendReadBuffer, responses.at(1), 1); + appendPayloadEditingID(s_backendReadBuffer, responses.at(2), 0); - expectedWriteBuffer = s_backendReadBuffer; + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(0).begin(), responses.at(0).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(1).begin(), responses.at(1).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(2).begin(), responses.at(2).end()); s_steps = { { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, @@ -3051,12 +3175,12 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* sending query (3) to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(2).size() }, /* backend is not ready yet, but the descriptor becomes ready */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { /* the backend descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); }}, /* nothing from the client */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData](int desc) { dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, /* backend closes the connection on us */ @@ -3077,7 +3201,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(2).size() }, /* sending the response (3) to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(2).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(2).size(), [&threadData](int desc) { /* the client descriptor becomes ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -3097,7 +3221,92 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); + IncomingTCPConnectionState::handleIO(state, now); + while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { + threadData.mplexer->run(&now); + } + + BOOST_CHECK_EQUAL(s_writeBuffer.size(), expectedWriteBuffer.size()); + BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); + BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); + BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(proxyEnabledBackend->outstanding.load(), 0U); + + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ + /* we should have nothing to clear since the connection cannot be reused due to the Proxy Protocol payload */ + BOOST_CHECK_EQUAL(IncomingTCPConnectionState::clearAllDownstreamConnections(), 0U); + } + + { + TEST_INIT("=> Outgoing proxy protocol, 3 queries to the backend, the client closes while sending the first response"); + + PacketBuffer expectedWriteBuffer; + PacketBuffer expectedBackendWriteBuffer; + + auto proxyPayload = makeProxyHeader(true, getBackendAddress("84", 4242), local, {}); + BOOST_REQUIRE_GT(proxyPayload.size(), s_proxyProtocolMinimumHeaderSize); + + s_readBuffer.insert(s_readBuffer.end(), queries.at(0).begin(), queries.at(0).end()); + s_readBuffer.insert(s_readBuffer.end(), queries.at(1).begin(), queries.at(1).end()); + s_readBuffer.insert(s_readBuffer.end(), queries.at(2).begin(), queries.at(2).end()); + + auto proxyEnabledBackend = std::make_shared(getBackendAddress("42", 53)); + proxyEnabledBackend->d_tlsCtx = tlsCtx; + /* enable out-of-order on the backend side as well */ + proxyEnabledBackend->d_maxInFlightQueriesPerConn = 65536; + proxyEnabledBackend->useProxyProtocol = true; + + expectedBackendWriteBuffer.insert(expectedBackendWriteBuffer.end(), proxyPayload.begin(), proxyPayload.end()); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), 1); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(2), 2); + + s_steps = { + { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, + /* reading a query from the client (1) */ + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(0).size() - 2 }, + /* opening a connection to the backend */ + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done }, + /* sending query (1) to the backend */ + { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, proxyPayload.size() + queries.at(0).size() }, + /* we try to read the response, not ready yet */ + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0 }, + /* reading a second query from the client */ + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(1).size() - 2 }, + /* sending query (2) to the backend */ + { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(1).size() }, + /* backend is not ready yet */ + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0 }, + /* reading a third query from the client */ + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(2).size() - 2 }, + /* sending query (3) to the backend */ + { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(2).size() }, + /* backend is not ready yet, but the descriptor becomes ready */ + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { + /* the backend descriptor becomes ready */ + dynamic_cast(threadData.mplexer.get())->setReady(desc); + }}, + /* client closes the connection */ + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 0 }, + /* closing the backend connection */ + { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done, 0 }, + /* closing the client connection */ + { ExpectedStep::ExpectedRequest::closeClient, IOState::Done, 0 }, + }; + + s_processQuery = [proxyEnabledBackend](DNSQuestion& dq, ClientState& cs, LocalHolders& holders, std::shared_ptr& selectedBackend) -> ProcessQueryResult { + selectedBackend = proxyEnabledBackend; + return ProcessQueryResult::PassToBackend; + }; + s_processResponse = [](PacketBuffer& response, LocalStateHolder >& localRespRuleActions, DNSResponse& dr, bool muted) -> bool { + return true; + }; + + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { threadData.mplexer->run(&now); @@ -3107,6 +3316,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(proxyEnabledBackend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ /* we should have nothing to clear since the connection cannot be reused due to the Proxy Protocol payload */ @@ -3142,7 +3352,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(1).size() - 2 }, /* reading a third query from the client */ { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, - { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(2).size() - 2, [&timeout](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(2).size() - 2, [&timeout](int desc) { timeout = true; } }, /* trying to read more from the client but nothing to read */ @@ -3161,7 +3371,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { threadData.mplexer->run(&now); @@ -3180,14 +3390,15 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK_EQUAL(s_writeBuffer.size(), 0U); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), 0U); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* restore */ backend->tcpSendTimeout = 30; g_tcpRecvTimeout = 2; /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ - /* we should have nothing to clear since the connection cannot be reused due to the Proxy Protocol payload */ - BOOST_CHECK_EQUAL(IncomingTCPConnectionState::clearAllDownstreamConnections(), 0U); + /* we have one connection to clear, no proxy protocol */ + BOOST_CHECK_EQUAL(IncomingTCPConnectionState::clearAllDownstreamConnections(), 1U); } { @@ -3198,17 +3409,31 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) for (const auto& query : queries) { s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); } - expectedBackendWriteBuffer = s_readBuffer; - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(0).begin(), responses.at(0).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(1).begin(), responses.at(1).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(2).begin(), responses.at(2).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(4).begin(), responses.at(4).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(3).begin(), responses.at(3).end()); + /* queries 0, 1 and 4 are sent to the first backend, 2 and 3 to the second */ + uint16_t firstBackendCounter = 0; + uint16_t secondBackendCounter = 0; + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), firstBackendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), firstBackendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(2), secondBackendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(3), secondBackendCounter++); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(4), firstBackendCounter++); + + firstBackendCounter = 0; + secondBackendCounter = 0; + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), firstBackendCounter++); + appendPayloadEditingID(s_backendReadBuffer, responses.at(1), firstBackendCounter++); + appendPayloadEditingID(s_backendReadBuffer, responses.at(2), secondBackendCounter++); + appendPayloadEditingID(s_backendReadBuffer, responses.at(4), firstBackendCounter++); + appendPayloadEditingID(s_backendReadBuffer, responses.at(3), secondBackendCounter++); - expectedWriteBuffer = s_backendReadBuffer; + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(0).begin(), responses.at(0).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(1).begin(), responses.at(1).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(2).begin(), responses.at(2).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(4).begin(), responses.at(4).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(3).begin(), responses.at(3).end()); - auto backend1 = std::make_shared(ComboAddress("192.0.2.42:53"), ComboAddress("0.0.0.0:0"), 0, std::string(), 1, false); + auto backend1 = std::make_shared(getBackendAddress("42", 53)); backend1->d_tlsCtx = tlsCtx; /* only two queries in flight! */ backend1->d_maxInFlightQueriesPerConn = 2; @@ -3222,7 +3447,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(0).size() - 2 }, /* opening a connection to the backend (1) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backend1Desc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backend1Desc](int desc) { backend1Desc = desc; } }, /* sending query (1) to the backend */ @@ -3240,7 +3465,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(2).size() - 2 }, /* opening a connection to the SECOND backend (2) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backend2Desc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backend2Desc](int desc) { backend2Desc = desc; } }, /* sending query (3) to backend 2 */ @@ -3255,7 +3480,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) /* no response ready yet */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0 }, /* nothing more to read from the client at that moment */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData, &backend1Desc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0, [&threadData, &backend1Desc](int desc) { /* but the first backend becomes readable */ dynamic_cast(threadData.mplexer.get())->setReady(backend1Desc); } }, @@ -3263,20 +3488,20 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData](int desc) { /* client becomes readable */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, /* reading a query from the client (5) */ { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, - { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(4).size() - 2, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(4).size() - 2, [&threadData](int desc) { /* client is not ready anymore */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, /* sending query (5) to the first backend (1) */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(4).size() }, /* no response ready yet, but the first backend becomes ready */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { /* set the outgoing descriptor (backend connection) as ready */ dynamic_cast(threadData.mplexer.get())->setReady(desc); } }, @@ -3286,7 +3511,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(1).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(1).size(), [&threadData,&backend1Desc,&backend2Desc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(1).size(), [&threadData,&backend1Desc,&backend2Desc](int desc) { /* client is NOT readable, backend1 is not readable, backend 2 becomes readable */ dynamic_cast(threadData.mplexer.get())->setNotReady(desc); dynamic_cast(threadData.mplexer.get())->setNotReady(backend1Desc); @@ -3296,7 +3521,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(2).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(2).size(), [&threadData,&backend1Desc,&backend2Desc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(2).size(), [&threadData,&backend1Desc,&backend2Desc](int desc) { /* backend 2 is no longer readable, backend 1 becomes readable */ dynamic_cast(threadData.mplexer.get())->setNotReady(backend2Desc); dynamic_cast(threadData.mplexer.get())->setReady(backend1Desc); @@ -3305,7 +3530,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(4).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size(), [&threadData,&backend1Desc,&backend2Desc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size(), [&threadData,&backend1Desc,&backend2Desc](int desc) { /* backend 1 is no longer readable, backend 2 becomes readable */ dynamic_cast(threadData.mplexer.get())->setNotReady(backend1Desc); dynamic_cast(threadData.mplexer.get())->setReady(backend2Desc); @@ -3314,7 +3539,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(3).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(3).size(), [&threadData,&backend2Desc](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(3).size(), [&threadData,&backend2Desc](int desc) { /* backend 2 is no longer readable */ dynamic_cast(threadData.mplexer.get())->setNotReady(backend2Desc); /* client becomes readable */ @@ -3337,7 +3562,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { threadData.mplexer->run(&now); @@ -3347,15 +3572,112 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), totalQueriesSize); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend1->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ BOOST_CHECK_EQUAL(IncomingTCPConnectionState::clearAllDownstreamConnections(), 2U); } + + { + TEST_INIT("=> 2 OOOR queries to the backend with duplicated IDs"); + PacketBuffer expectedWriteBuffer; + PacketBuffer expectedBackendWriteBuffer; + + s_readBuffer.insert(s_readBuffer.end(), queries.at(0).begin(), queries.at(0).end()); + s_readBuffer.insert(s_readBuffer.end(), queries.at(0).begin(), queries.at(0).end()); + + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), 1); + + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), 0); + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), 1); + + appendPayloadEditingID(expectedWriteBuffer, responses.at(0), 0); + appendPayloadEditingID(expectedWriteBuffer, responses.at(0), 0); + + bool timeout = false; + s_steps = { + { ExpectedStep::ExpectedRequest::handshakeClient, IOState::Done }, + /* reading a query from the client (1) */ + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(0).size() - 2 }, + /* opening a connection to the backend */ + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done }, + /* sending query to the backend */ + { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(0).size() }, + /* no response ready yet */ + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0 }, + /* reading a query from the client (2) */ + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, + { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(1).size() - 2 }, + /* sending query to the backend */ + { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(1).size() }, + /* no response ready yet, but mark the descriptor as ready */ + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData](int desc) { + dynamic_cast(threadData.mplexer.get())->setReady(desc); + } }, + /* nothing more from the client either */ + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0 }, + + /* reading response (1) from the backend */ + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() - 2 }, + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size()}, + /* sending it to the client */ + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size()}, + /* reading response (2) from the backend */ + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() - 2 }, + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size(), [&threadData](int desc) { + dynamic_cast(threadData.mplexer.get())->setNotReady(desc); + } }, + /* sending it to the client. we don't have anything else to send to the client, no new query from it either, until we time out */ + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&timeout](int desc) { + timeout = true; + } }, + /* closing client connection */ + { ExpectedStep::ExpectedRequest::closeClient, IOState::Done }, + /* closing a connection to the backend */ + { ExpectedStep::ExpectedRequest::closeBackend, IOState::Done }, + }; + + s_processQuery = [backend](DNSQuestion& dq, ClientState& cs, LocalHolders& holders, std::shared_ptr& selectedBackend) -> ProcessQueryResult { + selectedBackend = backend; + return ProcessQueryResult::PassToBackend; + }; + s_processResponse = [](PacketBuffer& response, LocalStateHolder >& localRespRuleActions, DNSResponse& dr, bool muted) -> bool { + return true; + }; + + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); + IncomingTCPConnectionState::handleIO(state, now); + while (!timeout && (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0)) { + threadData.mplexer->run(&now); + } + + struct timeval later = now; + later.tv_sec += g_tcpRecvTimeout + 1; + auto expiredConns = threadData.mplexer->getTimeouts(later); + BOOST_CHECK_EQUAL(expiredConns.size(), 1U); + for (const auto& cbData : expiredConns) { + if (cbData.second.type() == typeid(std::shared_ptr)) { + auto cbState = boost::any_cast>(cbData.second); + cbState->handleTimeout(cbState, false); + } + } + + BOOST_CHECK_EQUAL(s_writeBuffer.size(), expectedWriteBuffer.size()); + BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); + BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), expectedBackendWriteBuffer.size()); + BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); + + /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ + IncomingTCPConnectionState::clearAllDownstreamConnections(); + } } BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) { - ComboAddress local("192.0.2.1:80"); + auto local = getBackendAddress("1", 80); ClientState localCS(local, true, false, false, "", {}); /* enable out-of-order on the front side */ localCS.d_maxInFlightQueriesPerConn = 65536; @@ -3363,7 +3685,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) auto tlsCtx = std::make_shared(); localCS.tlsFrontend = std::make_shared(tlsCtx); - auto backend = std::make_shared(ComboAddress("192.0.2.42:53"), ComboAddress("0.0.0.0:0"), 0, std::string(), 1, false); + auto backend = std::make_shared(getBackendAddress("42", 53)); backend->d_tlsCtx = tlsCtx; /* shorter than the client one */ backend->tcpRecvTimeout = 1; @@ -3418,15 +3740,24 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) for (const auto& query : queries) { s_readBuffer.insert(s_readBuffer.end(), query.begin(), query.end()); } - expectedBackendWriteBuffer = s_readBuffer; - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(0).begin(), responses.at(0).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(2).begin(), responses.at(2).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(1).begin(), responses.at(1).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(4).begin(), responses.at(4).end()); - s_backendReadBuffer.insert(s_backendReadBuffer.end(), responses.at(3).begin(), responses.at(3).end()); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(0), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(1), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(2), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(3), 0); + appendPayloadEditingID(expectedBackendWriteBuffer, queries.at(4), 0); - expectedWriteBuffer = s_backendReadBuffer; + appendPayloadEditingID(s_backendReadBuffer, responses.at(0), 0); + appendPayloadEditingID(s_backendReadBuffer, responses.at(2), 0); + appendPayloadEditingID(s_backendReadBuffer, responses.at(1), 0); + appendPayloadEditingID(s_backendReadBuffer, responses.at(4), 0); + appendPayloadEditingID(s_backendReadBuffer, responses.at(3), 0); + + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(0).begin(), responses.at(0).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(2).begin(), responses.at(2).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(1).begin(), responses.at(1).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(4).begin(), responses.at(4).end()); + expectedWriteBuffer.insert(expectedWriteBuffer.end(), responses.at(3).begin(), responses.at(3).end()); std::vector backendDescriptors = { -1, -1, -1, -1, -1 }; @@ -3436,7 +3767,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(0).size() - 2 }, /* opening a connection to the backend (1) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc) { backendDescriptors.at(0) = desc; } }, /* sending query (1) to the backend */ @@ -3447,7 +3778,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(1).size() - 2 }, /* opening a connection to the backend (2) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc) { backendDescriptors.at(1) = desc; } }, /* sending query (2) to the backend */ @@ -3458,7 +3789,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(2).size() - 2 }, /* opening a connection to the backend (3) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc) { backendDescriptors.at(2) = desc; } }, /* sending query (3) to the backend */ @@ -3469,7 +3800,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(3).size() - 2 }, /* opening a connection to the backend (4) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc) { backendDescriptors.at(3) = desc; } }, /* sending query (3) to the backend */ @@ -3480,52 +3811,52 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromClient, IOState::Done, queries.at(4).size() - 2 }, /* opening a connection to the backend (5) */ - { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::connectToBackend, IOState::Done, 0, [&backendDescriptors](int desc) { backendDescriptors.at(4) = desc; } }, /* sending query (5) to the backend */ { ExpectedStep::ExpectedRequest::writeToBackend, IOState::Done, queries.at(4).size() }, /* no response ready yet, client stops being readable, first backend has a response */ - { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData,&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromBackend, IOState::NeedRead, 0, [&threadData,&backendDescriptors](int desc) { dynamic_cast(threadData.mplexer.get())->setReady(backendDescriptors.at(0)); } }, /* trying to read from the client but nothing yet */ - { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0 , [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::readFromClient, IOState::NeedRead, 0 , [&threadData](int desc) { dynamic_cast(threadData.mplexer.get())->setNotReady(desc); } }, /* reading response (1) from the first backend (1) */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(0).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData,&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(0).size(), [&threadData,&backendDescriptors](int desc) { dynamic_cast(threadData.mplexer.get())->setReady(backendDescriptors.at(2)); } }, /* reading response (3) from the third backend (3) */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(2).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(2).size(), [&threadData,&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(2).size(), [&threadData,&backendDescriptors](int desc) { dynamic_cast(threadData.mplexer.get())->setReady(backendDescriptors.at(1)); } }, /* reading response (2) from the second backend (2) */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(1).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(1).size(), [&threadData,&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(1).size(), [&threadData,&backendDescriptors](int desc) { dynamic_cast(threadData.mplexer.get())->setReady(backendDescriptors.at(4)); } }, /* reading response (5) from the fifth backend (5) */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(4).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size(), [&threadData,&backendDescriptors](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(4).size(), [&threadData,&backendDescriptors](int desc) { dynamic_cast(threadData.mplexer.get())->setReady(backendDescriptors.at(3)); } }, /* reading response (4) from the fourth backend (4) */ { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, 2 }, { ExpectedStep::ExpectedRequest::readFromBackend, IOState::Done, responses.at(3).size() - 2 }, /* sending it to the client */ - { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(3).size(), [&threadData](int desc, const ExpectedStep& step) { + { ExpectedStep::ExpectedRequest::writeToClient, IOState::Done, responses.at(3).size(), [&threadData](int desc) { dynamic_cast(threadData.mplexer.get())->setReady(-1); } }, /* client closes the connection */ @@ -3548,7 +3879,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) return true; }; - auto state = std::make_shared(ConnectionInfo(&localCS), threadData, now); + auto state = std::make_shared(ConnectionInfo(&localCS, getBackendAddress("84", 4242)), threadData, now); IncomingTCPConnectionState::handleIO(state, now); while (threadData.mplexer->getWatchedFDCount(false) != 0 || threadData.mplexer->getWatchedFDCount(true) != 0) { threadData.mplexer->run(&now); @@ -3558,6 +3889,7 @@ BOOST_AUTO_TEST_CASE(test_IncomingConnectionOOOR_BackendNotOOOR) BOOST_CHECK(s_writeBuffer == expectedWriteBuffer); BOOST_CHECK_EQUAL(s_backendWriteBuffer.size(), totalQueriesSize); BOOST_CHECK(s_backendWriteBuffer == expectedBackendWriteBuffer); + BOOST_CHECK_EQUAL(backend->outstanding.load(), 0U); /* we need to clear them now, otherwise we end up with dangling pointers to the steps via the TLS context, etc */ BOOST_CHECK_EQUAL(IncomingTCPConnectionState::clearAllDownstreamConnections(), 5U); diff --git a/pdns/dnsdistdist/tsan.supp b/pdns/dnsdistdist/tsan.supp index 2f9978b21f8f..7d289dba0efb 100644 --- a/pdns/dnsdistdist/tsan.supp +++ b/pdns/dnsdistdist/tsan.supp @@ -11,3 +11,4 @@ race:DownstreamState::setDown race:DownstreamState::setUp race:DownstreamState::setAuto race:updateHealthCheckResult +race:carbonDumpThread diff --git a/pdns/dnsgram.cc b/pdns/dnsgram.cc index a3bdc7d8d146..6cda63b3f073 100644 --- a/pdns/dnsgram.cc +++ b/pdns/dnsgram.cc @@ -169,24 +169,24 @@ try g_lastquestionTime=pr.d_pheader.ts; g_clientQuestions++; totalQueries++; - counts[make_pair(mdp.d_qname, mdp.d_qtype)]++; - questions.insert(make_pair(mdp.d_qname, mdp.d_qtype)); + counts[pair(mdp.d_qname, mdp.d_qtype)]++; + questions.emplace(mdp.d_qname, mdp.d_qtype); } else if(mdp.d_header.rd && mdp.d_header.qr) { rdacounts[pr.d_pheader.ts.tv_sec + 0.01*(pr.d_pheader.ts.tv_usec/10000)]++; g_lastanswerTime=pr.d_pheader.ts; g_clientResponses++; - answers.insert(make_pair(mdp.d_qname, mdp.d_qtype)); + answers.emplace(mdp.d_qname, mdp.d_qtype); } else if(!mdp.d_header.rd && !mdp.d_header.qr) { g_lastquestionTime=pr.d_pheader.ts; g_serverQuestions++; - counts[make_pair(mdp.d_qname, mdp.d_qtype)]++; - questions.insert(make_pair(mdp.d_qname, mdp.d_qtype)); + counts[pair(mdp.d_qname, mdp.d_qtype)]++; + questions.emplace(mdp.d_qname, mdp.d_qtype); totalQueries++; } else if(!mdp.d_header.rd && mdp.d_header.qr) { - answers.insert(make_pair(mdp.d_qname, mdp.d_qtype)); + answers.emplace(mdp.d_qname, mdp.d_qtype); g_serverResponses++; } @@ -233,7 +233,7 @@ try ofstream failed("failed"); failed<<"name\ttype\tnumber\n"; for(diff_t::const_iterator i = diff.begin(); i != diff.end() ; ++i) { - failed << i->first << "\t" << DNSRecordContent::NumberToType(i->second) << "\t"<< counts[make_pair(i->first, i->second)]<<"\n"; + failed << i->first << "\t" << DNSRecordContent::NumberToType(i->second) << "\t"<< counts[pair(i->first, i->second)]<<"\n"; } diff.clear(); @@ -245,7 +245,7 @@ try ofstream succeeded("succeeded"); succeeded<<"name\ttype\tnumber\n"; for(queries_t::const_iterator i = answers.begin(); i != answers.end() ; ++i) { - succeeded << i->first << "\t" <second) << "\t" << counts[make_pair(i->first, i->second)]<<"\n"; + succeeded << i->first << "\t" <second) << "\t" << counts[pair(i->first, i->second)]<<"\n"; } } } diff --git a/pdns/dnsmessage.proto b/pdns/dnsmessage.proto index 972bd8de38f2..ef6b77436797 100644 --- a/pdns/dnsmessage.proto +++ b/pdns/dnsmessage.proto @@ -1,25 +1,36 @@ /* * This file describes the message format used by the protobuf logging feature in PowerDNS and dnsdist. * - * Written by PowerDNS.COM B.V. and its contributors. + * MIT License * - * To the extent possible under law, the author(s) have dedicated all - * copyright and related and neighboring rights to this file to the public - * domain worldwide. This file is distributed without any warranty. - * - * You should have received a copy of the CC0 Public Domain Dedication along - * with this file. If not, see: - * - * . + * Copyright (c) 2016-now PowerDNS.COM B.V. and its contributors. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ syntax = "proto2"; message PBDNSMessage { enum Type { - DNSQueryType = 1; - DNSResponseType = 2; - DNSOutgoingQueryType = 3; - DNSIncomingResponseType = 4; + DNSQueryType = 1; // Query received by the service + DNSResponseType = 2; // Response returned by the service + DNSOutgoingQueryType = 3; // Query sent out by the service to a remote server + DNSIncomingResponseType = 4; // Response returned by the remote server } enum SocketFamily { INET = 1; // IPv4 (RFC 791) @@ -28,9 +39,13 @@ message PBDNSMessage { enum SocketProtocol { UDP = 1; // User Datagram Protocol (RFC 768) TCP = 2; // Transmission Control Protocol (RFC 793) + DOT = 3; // DNS over TLS (RFC 7858) + DOH = 4; // DNS over HTTPS (RFC 8484) + DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol) + DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol) } enum PolicyType { - UNKNOWN = 1; // No policy applied, or unknown type + UNKNOWN = 1; // No RPZ policy applied, or unknown type QNAME = 2; // Policy matched on the QName CLIENTIP = 3; // Policy matched on the client IP RESPONSEIP = 4; // Policy matched on one of the IPs contained in the answer @@ -65,52 +80,54 @@ message PBDNSMessage { BogusRevokedDNSKEY = 17; BogusInvalidDNSKEYProtocol = 18; } - required Type type = 1; + required Type type = 1; // Type of event optional bytes messageId = 2; // UUID, shared by the query and the response optional bytes serverIdentity = 3; // ID of the server emitting the protobuf message optional SocketFamily socketFamily = 4; optional SocketProtocol socketProtocol = 5; - optional bytes from = 6; // DNS requestor (client) - optional bytes to = 7; // DNS responder (server) + optional bytes from = 6; // DNS requestor (client) as 4 (IPv4) or 16 (IPv6) raw bytes in network byte order + optional bytes to = 7; // DNS responder (server) as 4 (IPv4) or 16 (IPv6) raw bytes in network byte order optional uint64 inBytes = 8; // Size of the query or response on the wire optional uint32 timeSec = 9; // Time of message reception (seconds since epoch) optional uint32 timeUsec = 10; // Time of message reception (additional micro-seconds) optional uint32 id = 11; // ID of the query/response as found in the DNS header message DNSQuestion { - optional string qName = 1; - optional uint32 qType = 2; - optional uint32 qClass = 3; + optional string qName = 1; // Fully qualified DNS name (with trailing dot) + optional uint32 qType = 2; // https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 + optional uint32 qClass = 3; // Typically 1 (IN), see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2 } - optional DNSQuestion question = 12; + optional DNSQuestion question = 12; // DNS query received from client message DNSResponse { + // See exportTypes in https://docs.powerdns.com/recursor/lua-config/protobuf.html#protobufServer + // for the list of supported resource record types. message DNSRR { - optional string name = 1; - optional uint32 type = 2; - optional uint32 class = 3; - optional uint32 ttl = 4; - optional bytes rdata = 5; + optional string name = 1; // Fully qualified DNS name (with trailing dot) + optional uint32 type = 2; // https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4 + optional uint32 class = 3; // Typically 1 (IN), see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2 + optional uint32 ttl = 4; // TTL in seconds + optional bytes rdata = 5; // raw address bytes in network byte order for A & AAAA; text representation for others, with fully qualified (trailing dot) domain names optional bool udr = 6; // True if this is the first time this RR has been seen for this question } optional uint32 rcode = 1; // DNS Response code, or 65536 for a network error including a timeout - repeated DNSRR rrs = 2; + repeated DNSRR rrs = 2; // DNS resource records in response optional string appliedPolicy = 3; // Filtering policy (RPZ or Lua) applied - repeated string tags = 4; // Additional tags + repeated string tags = 4; // Additional tags applied optional uint32 queryTimeSec = 5; // Time of the corresponding query reception (seconds since epoch) optional uint32 queryTimeUsec = 6; // Time of the corresponding query reception (additional micro-seconds) optional PolicyType appliedPolicyType = 7; // Type of the filtering policy (RPZ or Lua) applied optional string appliedPolicyTrigger = 8; // The RPZ trigger optional string appliedPolicyHit = 9; // The value (qname or IP) that caused the hit optional PolicyKind appliedPolicyKind = 10; // The Kind (RPZ action) applied by the hit - optional VState validationState = 11; // The DNSSEC Validation State + optional VState validationState = 11; // The DNSSEC Validation State } optional DNSResponse response = 13; - optional bytes originalRequestorSubnet = 14; // EDNS Client Subnet value + optional bytes originalRequestorSubnet = 14; // EDNS Client Subnet value (4 or 16 raw bytes in network byte order) optional string requestorId = 15; // Username of the requestor optional bytes initialRequestId = 16; // UUID of the incoming query that initiated this outgoing query or incoming response - optional bytes deviceId = 17; // Device ID of the requestor (could be mac address IP address or e.g. IMEI) + optional bytes deviceId = 17; // Device ID of the requestor (could be mac address IP address or e.g. IMEI, format implementation dependent) optional bool newlyObservedDomain = 18; // True if the domain has not been seen before optional string deviceName = 19; // Device name of the requestor optional uint32 fromPort = 20; // Source port of the DNS query (client) @@ -126,6 +143,39 @@ message PBDNSMessage { required MetaValue value = 2; } repeated Meta meta = 22; // Arbitrary meta-data - to be used in future rather than adding new fields all the time + + // The well known EventTrace event numbers + enum EventType { + // Range 0..99: Generic events + CustomEvent = 0; // A custom event + ReqRecv = 1; // A request was received + PCacheCheck = 2; // A packet cache check was initiated or completed; value: bool cacheHit + AnswerSent = 3; // An answer was sent to the client + + // Range 100: Recursor events + SyncRes = 100; // Recursor Syncres main function has started or completed; value: int rcode + LuaGetTag = 101; // Events below mark start or end of Lua hook calls; value: return value of hook + LuaGetTagFFI = 102; + LuaIPFilter = 103; + LuaPreRPZ = 104; + LuaPreResolve = 105; + LuaPreOutQuery = 106; + LuaPostResolve = 107; + LuaNoData = 108; + LuaNXDomain = 109; + } + + message Event { + required int64 ts = 1; // Timestamp in ns relative to time of creation of event trace data structure + required EventType event = 2; // Type of event + required bool start = 3; // true for "start" events, false for "completed" events + optional bool boolVal = 4; // Below are optional values associated with events + optional int64 intVal = 5; + optional string stringVal = 6; + optional bytes bytesVal = 7; + optional string custom = 8; // The name of the event for custom events + } + repeated Event trace = 23; } message PBDNSMessageList { diff --git a/pdns/dnspacket.cc b/pdns/dnspacket.cc index 705956c2f52b..b41949ce217d 100644 --- a/pdns/dnspacket.cc +++ b/pdns/dnspacket.cc @@ -38,6 +38,7 @@ #include "dns.hh" #include "dnsbackend.hh" #include "ednsoptions.hh" +#include "ednscookies.hh" #include "pdnsexception.hh" #include "dnspacket.hh" #include "logger.hh" @@ -52,6 +53,8 @@ #include "shuffle.hh" bool DNSPacket::s_doEDNSSubnetProcessing; +bool DNSPacket::s_doEDNSCookieProcessing; +string DNSPacket::s_EDNSCookieKey; uint16_t DNSPacket::s_udpTruncationThreshold; DNSPacket::DNSPacket(bool isQuery): d_isQuery(isQuery) @@ -67,11 +70,52 @@ const string& DNSPacket::getString() return d_rawpacket; } +string DNSPacket::getRemoteString() const +{ + string ret; + + ret = getRemote().toString(); + + if (d_inner_remote) { + ret += "(" + d_inner_remote->toString() + ")"; + } + + if(hasEDNSSubnet()) { + ret += "<-" + getRealRemote().toString(); + } + + return ret; +} + +string DNSPacket::getRemoteStringWithPort() const +{ + string ret; + + ret = getRemote().toStringWithPort(); + + if (d_inner_remote) { + ret += "(" + d_inner_remote->toStringWithPort() + ")"; + } + + if(hasEDNSSubnet()) { + ret += "<-" + getRealRemote().toString(); + } + + return ret; +} + ComboAddress DNSPacket::getRemote() const { return d_remote; } +ComboAddress DNSPacket::getInnerRemote() const +{ + if (d_inner_remote) + return *d_inner_remote; + return d_remote; +} + uint16_t DNSPacket::getRemotePort() const { return d_remote.sin4.sin_port; @@ -195,7 +239,8 @@ void DNSPacket::setCompress(bool compress) bool DNSPacket::couldBeCached() const { - return !d_wantsnsid && qclass==QClass::IN && !d_havetsig; + return !d_wantsnsid && qclass==QClass::IN && !d_havetsig && + !(d_haveednscookie && s_doEDNSCookieProcessing); } unsigned int DNSPacket::getMinTTL() @@ -273,7 +318,7 @@ void DNSPacket::wrapup() if(d_wantsnsid) { const static string mode_server_id=::arg()["server-id"]; if(mode_server_id != "disabled") { - opts.push_back(make_pair(EDNSOptionCode::NSID, mode_server_id)); + opts.emplace_back(EDNSOptionCode::NSID, mode_server_id); optsize += EDNS_OPTION_CODE_SIZE + EDNS_OPTION_LENGTH_SIZE + mode_server_id.size(); } } @@ -285,6 +330,12 @@ void DNSPacket::wrapup() optsize += d_eso.source.isIPv4() ? 4 : 16; } + if (d_haveednscookie) { + if (d_eco.isWellFormed()) { + optsize += EDNSCookiesOpt::EDNSCookieOptSize; + } + } + if (d_trc.d_algoName.countLabels()) { // TSIG is not OPT, but we count it in optsize anyway @@ -294,7 +345,7 @@ void DNSPacket::wrapup() static_assert(EVP_MAX_MD_SIZE <= 64, "EVP_MAX_MD_SIZE is overly huge on this system, please check"); } - if(!d_rrs.empty() || !opts.empty() || d_haveednssubnet || d_haveednssection) { + if(!d_rrs.empty() || !opts.empty() || d_haveednssubnet || d_haveednssection || d_haveednscookie) { try { uint8_t maxScopeMask=0; for(pos=d_rrs.begin(); pos < d_rrs.end(); ++pos) { @@ -322,7 +373,12 @@ void DNSPacket::wrapup() eso.scope = Netmask(eso.source.getNetwork(), maxScopeMask); string opt = makeEDNSSubnetOptsString(eso); - opts.push_back(make_pair(8, opt)); // 'EDNS SUBNET' + opts.emplace_back(8, opt); // 'EDNS SUBNET' + } + + if (d_haveednscookie && d_eco.isWellFormed()) { + d_eco.makeServerCookie(s_EDNSCookieKey, getInnerRemote()); + opts.emplace_back(EDNSOptionCode::COOKIE, d_eco.makeOptString()); } if(!opts.empty() || d_haveednssection || d_dnssecOk) @@ -369,6 +425,7 @@ std::unique_ptr DNSPacket::replyPacket() const r->setSocket(d_socket); r->d_anyLocal=d_anyLocal; r->setRemote(&d_remote); + r->d_inner_remote=d_inner_remote; r->setAnswer(true); // this implies the allocation of the header r->setA(true); // and we are authoritative r->setRA(false); // no recursion available @@ -386,8 +443,10 @@ std::unique_ptr DNSPacket::replyPacket() const r->d_wantsnsid = d_wantsnsid; r->d_dnssecOk = d_dnssecOk; r->d_eso = d_eso; + r->d_eco = d_eco; r->d_haveednssubnet = d_haveednssubnet; r->d_haveednssection = d_haveednssection; + r->d_haveednscookie = d_haveednscookie; r->d_ednsversion = 0; r->d_ednsrcode = 0; @@ -423,7 +482,7 @@ int DNSPacket::noparse(const char *mesg, size_t length) d_rawpacket.assign(mesg,length); if(length < 12) { g_log << Logger::Debug << "Ignoring packet: too short ("<first<<": "<second)< #include #include #include @@ -34,6 +33,7 @@ #include #include +#include #include #include #include "qtype.hh" @@ -61,8 +61,9 @@ public: const string& getString(); //!< for serialization - just passes the whole packet // address & socket manipulation - void setRemote(const ComboAddress*); + void setRemote(const ComboAddress*, std::optional = std::nullopt); ComboAddress getRemote() const; + ComboAddress getInnerRemote() const; // for proxy protocol Netmask getRealRemote() const; ComboAddress getLocal() const { @@ -73,6 +74,9 @@ public: } uint16_t getRemotePort() const; + string getRemoteString() const; + string getRemoteStringWithPort() const; + boost::optional d_anyLocal; Utility::sock_t getSocket() const @@ -81,7 +85,6 @@ public: } void setSocket(Utility::sock_t sock); - // these manipulate 'd' void setA(bool); //!< make this packet authoritative - manipulates 'd' void setID(uint16_t); //!< set the DNS id of this packet - manipulates 'd' @@ -121,6 +124,9 @@ public: bool couldBeCached() const; //!< returns 0 if this query should bypass the packet cache bool hasEDNSSubnet() const; bool hasEDNS() const; + bool hasEDNSCookie() const; + bool hasWellFormedEDNSCookie() const; + bool hasValidEDNSCookie() const; uint8_t getEDNSVersion() const { return d_ednsversion; }; void setEDNSRcode(uint16_t extRCode) { @@ -144,6 +150,7 @@ public: TSIGRecordContent d_trc; //72 ComboAddress d_remote; //28 + std::optional d_inner_remote; // the 'outer' remote is the IP on the physical packet header. The 'inner' remote lives one layer deeper, in the PROXY header. TSIGHashEnum d_tsig_algo{TSIG_MD5}; //4 int d_ednsRawPacketSizeLimit{-1}; // only used for Lua record @@ -163,6 +170,8 @@ public: static uint16_t s_udpTruncationThreshold; static bool s_doEDNSSubnetProcessing; + static bool s_doEDNSCookieProcessing; + static string s_EDNSCookieKey; private: void pasteQ(const char *question, int length); //!< set the question of this packet, useful for crafting replies @@ -175,6 +184,7 @@ private: std::unordered_set d_dedup; string d_rawpacket; // this is where everything lives 8 EDNSSubnetOpts d_eso; + EDNSCookiesOpt d_eco; int d_maxreplylen{0}; int d_socket{-1}; // 4 @@ -188,6 +198,8 @@ private: bool d_tsigtimersonly{false}; bool d_wantsnsid{false}; bool d_haveednssubnet{false}; + bool d_haveednscookie{false}; + bool d_ednscookievalid{false}; bool d_haveednssection{false}; bool d_isQuery; }; diff --git a/pdns/dnsparser.cc b/pdns/dnsparser.cc index 10327144841f..57267218db87 100644 --- a/pdns/dnsparser.cc +++ b/pdns/dnsparser.cc @@ -124,7 +124,7 @@ std::shared_ptr DNSRecordContent::mastermake(const DNSRecord & { uint16_t searchclass = (dr.d_type == QType::OPT) ? 1 : dr.d_class; // class is invalid for OPT - typemap_t::const_iterator i=getTypemap().find(make_pair(searchclass, dr.d_type)); + auto i = getTypemap().find(pair(searchclass, dr.d_type)); if(i==getTypemap().end() || !i->second) { return std::make_shared(dr, pr); } @@ -135,7 +135,7 @@ std::shared_ptr DNSRecordContent::mastermake(const DNSRecord & std::shared_ptr DNSRecordContent::mastermake(uint16_t qtype, uint16_t qclass, const string& content) { - zmakermap_t::const_iterator i=getZmakermap().find(make_pair(qclass, qtype)); + auto i = getZmakermap().find(pair(qclass, qtype)); if(i==getZmakermap().end()) { return std::make_shared(content); } @@ -152,7 +152,7 @@ std::shared_ptr DNSRecordContent::mastermake(const DNSRecord & uint16_t searchclass = (dr.d_type == QType::OPT) ? 1 : dr.d_class; // class is invalid for OPT - typemap_t::const_iterator i=getTypemap().find(make_pair(searchclass, dr.d_type)); + auto i = getTypemap().find(pair(searchclass, dr.d_type)); if(i==getTypemap().end() || !i->second) { return std::make_shared(dr, pr); } @@ -298,7 +298,7 @@ void MOADNSParser::init(bool query, const pdns_string_view& packet) d_tsigPos = recordStartPos; } - d_answers.push_back(make_pair(std::move(dr), pr.getPosition() - sizeof(dnsheader))); + d_answers.emplace_back(std::move(dr), pr.getPosition() - sizeof(dnsheader)); } #if 0 diff --git a/pdns/dnsparser.hh b/pdns/dnsparser.hh index 80732d6c587a..9cbe5a4e2bac 100644 --- a/pdns/dnsparser.hh +++ b/pdns/dnsparser.hh @@ -233,17 +233,17 @@ public: static void regist(uint16_t cl, uint16_t ty, makerfunc_t* f, zmakerfunc_t* z, const char* name) { if(f) - getTypemap()[make_pair(cl,ty)]=f; + getTypemap()[pair(cl,ty)]=f; if(z) - getZmakermap()[make_pair(cl,ty)]=z; + getZmakermap()[pair(cl,ty)]=z; - getT2Namemap().insert(make_pair(make_pair(cl,ty), name)); - getN2Typemap().insert(make_pair(name, make_pair(cl,ty))); + getT2Namemap().emplace(pair(cl, ty), name); + getN2Typemap().emplace(name, pair(cl, ty)); } static void unregist(uint16_t cl, uint16_t ty) { - pair key=make_pair(cl, ty); + auto key = pair(cl, ty); getTypemap().erase(key); getZmakermap().erase(key); } @@ -267,7 +267,7 @@ public: static const string NumberToType(uint16_t num, uint16_t classnum=1) { - t2namemap_t::const_iterator iter = getT2Namemap().find(make_pair(classnum, num)); + auto iter = getT2Namemap().find(pair(classnum, num)); if(iter == getT2Namemap().end()) return "TYPE" + std::to_string(num); // throw runtime_error("Unknown DNS type with numerical id "+std::to_string(num)); @@ -451,7 +451,7 @@ class DNSPacketMangler { public: explicit DNSPacketMangler(std::string& packet) - : d_packet((char*) packet.c_str()), d_length(packet.length()), d_notyouroffset(12), d_offset(d_notyouroffset) + : d_packet(packet.data()), d_length(packet.length()), d_notyouroffset(12), d_offset(d_notyouroffset) {} DNSPacketMangler(char* packet, size_t length) : d_packet(packet), d_length(length), d_notyouroffset(12), d_offset(d_notyouroffset) @@ -486,7 +486,7 @@ public: const char* p = d_packet + d_offset; moveOffset(4); uint32_t ret; - memcpy(&ret, (void*)p, sizeof(ret)); + memcpy(&ret, p, sizeof(ret)); return ntohl(ret); } uint16_t get16BitInt() @@ -494,7 +494,7 @@ public: const char* p = d_packet + d_offset; moveOffset(2); uint16_t ret; - memcpy(&ret, (void*)p, sizeof(ret)); + memcpy(&ret, p, sizeof(ret)); return ntohs(ret); } @@ -517,7 +517,7 @@ public: moveOffset(4); uint32_t tmp; - memcpy(&tmp, (void*) p, sizeof(tmp)); + memcpy(&tmp, p, sizeof(tmp)); tmp = ntohl(tmp); if (tmp > decrease) { tmp -= decrease; diff --git a/pdns/dnspcap2protobuf.cc b/pdns/dnspcap2protobuf.cc index bf413edcc4dc..e5ff300d3d0f 100644 --- a/pdns/dnspcap2protobuf.cc +++ b/pdns/dnspcap2protobuf.cc @@ -104,7 +104,7 @@ try { queryTime.tv_sec = pr.d_pheader.ts.tv_sec; queryTime.tv_usec = pr.d_pheader.ts.tv_usec; uniqueId = getUniqueID(); - ids[dh->id] = std::make_pair(uniqueId, queryTime); + ids[dh->id] = {uniqueId, queryTime}; } else { const auto& it = ids.find(dh->id); @@ -126,7 +126,7 @@ try { pbBuffer.clear(); pdns::ProtoZero::Message pbMessage(pbBuffer); pbMessage.setType(dh->qr ? pdns::ProtoZero::Message::MessageType::DNSResponseType : pdns::ProtoZero::Message::MessageType::DNSQueryType); - pbMessage.setRequest(uniqueId, requestor, responder, qname, qtype, qclass, dh->id, false, pr.d_len); + pbMessage.setRequest(uniqueId, requestor, responder, qname, qtype, qclass, dh->id, pdns::ProtoZero::Message::TransportProtocol::UDP, pr.d_len); pbMessage.setTime(pr.d_pheader.ts.tv_sec, pr.d_pheader.ts.tv_usec); if (dh->qr) { diff --git a/pdns/dnsproxy.cc b/pdns/dnsproxy.cc index db07c2a6bbd0..179a8c9aabb7 100644 --- a/pdns/dnsproxy.cc +++ b/pdns/dnsproxy.cc @@ -40,7 +40,7 @@ extern StatBag S; -DNSProxy::DNSProxy(const string &remote) +DNSProxy::DNSProxy(const string &remote): d_xor(dns_random_uint16()) { d_resanswers=S.getPointer("recursing-answers"); d_resquestions=S.getPointer("recursing-questions"); @@ -79,7 +79,6 @@ DNSProxy::DNSProxy(const string &remote) throw PDNSException("Unable to UDP connect to remote nameserver "+d_remote.toStringWithPort()+": "+stringerror()); } - d_xor=dns_random_uint16(); g_log<& r, const DNSName& targ uint16_t id; uint16_t qtype = r->qtype.getCode(); { - std::lock_guard l(d_lock); - id=getID_locked(); + auto conntrack = d_conntrack.lock(); + id = getID_locked(*conntrack); ConntrackEntry ce; ce.id = r->d.id; @@ -145,7 +144,7 @@ bool DNSProxy::completePacket(std::unique_ptr& r, const DNSName& targ ce.complete = std::move(r); ce.aname=aname; ce.anameScopeMask = scopeMask; - d_conntrack[id]=std::move(ce); + (*conntrack)[id]=std::move(ce); } vector packet; @@ -163,12 +162,12 @@ bool DNSProxy::completePacket(std::unique_ptr& r, const DNSName& targ /** This finds us an unused or stale ID. Does not actually clean the contents */ -int DNSProxy::getID_locked() +int DNSProxy::getID_locked(map_t& conntrack) { map_t::iterator i; for(int n=0;;++n) { - i=d_conntrack.find(n); - if(i==d_conntrack.end()) { + i=conntrack.find(n); + if(i==conntrack.end()) { return n; } else if(i->second.created l(d_lock); + auto conntrack = d_conntrack.lock(); #if BYTE_ORDER == BIG_ENDIAN // this is needed because spoof ID down below does not respect the native byteorder d.id = ( 256 * (uint16_t)buffer[1] ) + (uint16_t)buffer[0]; #endif - map_t::iterator i=d_conntrack.find(d.id^d_xor); - if(i==d_conntrack.end()) { + map_t::iterator i=conntrack->find(d.id^d_xor); + if(i==conntrack->end()) { g_log< pw=nullptr; if(!g_vm["write-failures"].as().empty()) - pw=std::unique_ptr(new PcapPacketWriter(g_vm["write-failures"].as(), pr)); - + pw = std::make_unique(g_vm["write-failures"].as(), pr); + EDNSOpts edo; while(pr.getUDPPacket()) { @@ -283,12 +283,11 @@ try lc.outstanding = liveQuestions(); LiveCounts diff = lc - lastcounts; - pcounts.push_back(make_pair(pr.d_pheader.ts.tv_sec, diff)); - - } - lastsec = pr.d_pheader.ts.tv_sec; - lastcounts = lc; - } + pcounts.emplace_back(pr.d_pheader.ts.tv_sec, diff); + } + lastsec = pr.d_pheader.ts.tv_sec; + lastcounts = lc; + } if(lowestTime) { lowestTime = min((time_t)lowestTime, (time_t)pr.d_pheader.ts.tv_sec); } else { lowestTime = pr.d_pheader.ts.tv_sec; } diff --git a/pdns/dnssecinfra.cc b/pdns/dnssecinfra.cc index e2db27f07ca6..df008fc97a43 100644 --- a/pdns/dnssecinfra.cc +++ b/pdns/dnssecinfra.cc @@ -205,7 +205,7 @@ vector> DNSCryptoKeyEngine::listAllAlgosWithBackend() vector> ret; for (auto const& value : getMakers()) { auto dcke = value.second(value.first); - ret.push_back(make_pair(value.first, dcke->getName())); + ret.emplace_back(value.first, dcke->getName()); } return ret; } @@ -632,27 +632,6 @@ static string calculateHMAC(const std::string& key, const std::string& text, TSI return string((char*) hash, outlen); } -static bool constantTimeStringEquals(const std::string& a, const std::string& b) -{ - if (a.size() != b.size()) { - return false; - } - const size_t size = a.size(); -#ifdef HAVE_CRYPTO_MEMCMP - return CRYPTO_memcmp(a.c_str(), b.c_str(), size) == 0; -#else - const volatile unsigned char *_a = (const volatile unsigned char *) a.c_str(); - const volatile unsigned char *_b = (const volatile unsigned char *) b.c_str(); - unsigned char res = 0; - - for (size_t idx = 0; idx < size; idx++) { - res |= _a[idx] ^ _b[idx]; - } - - return res == 0; -#endif -} - static string makeTSIGPayload(const string& previous, const char* packetBegin, size_t packetSize, const DNSName& tsigKeyName, const TSIGRecordContent& trc, bool timersonly) { string message; diff --git a/pdns/dnsseckeeper.hh b/pdns/dnsseckeeper.hh index edae2152f7a3..29f2fd0cbb89 100644 --- a/pdns/dnsseckeeper.hh +++ b/pdns/dnsseckeeper.hh @@ -296,11 +296,9 @@ private: void cleanup(); - static keycache_t s_keycache; - static metacache_t s_metacache; + static SharedLockGuarded s_keycache; + static SharedLockGuarded s_metacache; static int64_t s_metaCacheCleanActions; - static ReadWriteLock s_metacachelock; - static ReadWriteLock s_keycachelock; static AtomicCounter s_ops; static time_t s_last_prune; static size_t s_maxEntries; diff --git a/pdns/dnssecsigner.cc b/pdns/dnssecsigner.cc index a98acf72959f..3536902ca481 100644 --- a/pdns/dnssecsigner.cc +++ b/pdns/dnssecsigner.cc @@ -33,9 +33,8 @@ #include "statbag.hh" extern StatBag S; -static ReadWriteLock g_signatures_lock; typedef map, string> signaturecache_t; -static signaturecache_t g_signatures; +static SharedLockGuarded g_signatures; static int g_cacheweekno; const static std::set g_KSKSignedQTypes {QType::DNSKEY, QType::CDS, QType::CDNSKEY}; @@ -67,9 +66,9 @@ static void fillOutRRSIG(DNSSECPrivateKey& dpk, const DNSName& signQName, RRSIGR bool doCache=true; if(doCache) { - ReadLock l(&g_signatures_lock); - signaturecache_t::const_iterator iter = g_signatures.find(lookup); - if(iter != g_signatures.end()) { + auto signatures = g_signatures.read_lock(); + signaturecache_t::const_iterator iter = signatures->find(lookup); + if (iter != signatures->end()) { rrc.d_signature=iter->second; return; } @@ -83,13 +82,13 @@ static void fillOutRRSIG(DNSSECPrivateKey& dpk, const DNSName& signQName, RRSIGR int weekno = (time(nullptr) - dns_random(3600)) / (86400*7); // we just spent milliseconds doing a signature, microsecond more won't kill us const static int maxcachesize=::arg().asNum("max-signature-cache-entries", INT_MAX); - WriteLock l(&g_signatures_lock); - if(g_cacheweekno < weekno || g_signatures.size() >= (uint) maxcachesize) { // blunt but effective (C) Habbie, mind04 + auto signatures = g_signatures.write_lock(); + if (g_cacheweekno < weekno || signatures->size() >= (uint) maxcachesize) { // blunt but effective (C) Habbie, mind04 g_log<clear(); g_cacheweekno = weekno; } - g_signatures[lookup] = rrc.d_signature; + (*signatures)[lookup] = rrc.d_signature; } } @@ -169,8 +168,7 @@ static void addSignature(DNSSECKeeper& dk, UeberBackend& db, const DNSName& sign uint64_t signatureCacheSize(const std::string& str) { - ReadLock l(&g_signatures_lock); - return g_signatures.size(); + return g_signatures.read_lock()->size(); } static bool rrsigncomp(const DNSZoneRecord& a, const DNSZoneRecord& b) diff --git a/pdns/dnstap.hh b/pdns/dnstap.hh index f4c7ee6ad959..798e86762eb5 100644 --- a/pdns/dnstap.hh +++ b/pdns/dnstap.hh @@ -34,7 +34,7 @@ class DnstapMessage { public: enum class MessageType : uint32_t { auth_query = 1, auth_response = 2, resolver_query = 3, resolver_response = 4, client_query = 5, client_response = 6, forwarder_query = 7, forwarded_response = 8, stub_query = 9, stub_response = 10, tool_query = 11, tool_response = 12 }; - enum class ProtocolType : uint32_t { DoUDP = 1, DoTCP = 2, DoT = 3, DoH = 4 }; + enum class ProtocolType : uint32_t { DoUDP = 1, DoTCP = 2, DoT = 3, DoH = 4, DNSCryptUDP = 5, DNSCryptTCP = 6 }; DnstapMessage(std::string& buffer, MessageType type, const std::string& identity, const ComboAddress* requestor, const ComboAddress* responder, ProtocolType protocol, const char* packet, const size_t len, const struct timespec* queryTime, const struct timespec* responseTime, boost::optional auth=boost::none); diff --git a/pdns/dnstap.proto b/pdns/dnstap.proto index 1aca53f02a6d..3780c4934f2b 100644 --- a/pdns/dnstap.proto +++ b/pdns/dnstap.proto @@ -58,10 +58,12 @@ enum SocketFamily { // SocketProtocol: the protocol used to transport a DNS message. enum SocketProtocol { - UDP = 1; // DNS over UDP transport (RFC 1035 section 4.2.1) - TCP = 2; // DNS over TCP transport (RFC 1035 section 4.2.2) - DOT = 3; // DNS over TLS (RFC 7858) - DOH = 4; // DNS over HTTPS (RFC 8484) + UDP = 1; // DNS over UDP transport (RFC 1035 section 4.2.1) + TCP = 2; // DNS over TCP transport (RFC 1035 section 4.2.2) + DOT = 3; // DNS over TLS (RFC 7858) + DOH = 4; // DNS over HTTPS (RFC 8484) + DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol) + DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol) } // Policy: information about any name server operator policy diff --git a/pdns/dnstcpbench.cc b/pdns/dnstcpbench.cc index aa461ac18d96..51f1e8b3cefd 100644 --- a/pdns/dnstcpbench.cc +++ b/pdns/dnstcpbench.cc @@ -134,7 +134,7 @@ try throw PDNSException("tcp read failed"); len=ntohs(len); - std::unique_ptr creply(new char[len]); + auto creply = std::make_unique(len); int n=0; int numread; while(n make() { - return std::unique_ptr(new IPSeqObfuscator()); + return std::make_unique(); } uint32_t obf4(uint32_t orig) override @@ -133,7 +133,7 @@ class IPCipherObfuscator : public IPObfuscator {} static std::unique_ptr make(std::string key, bool decrypt) { - return std::unique_ptr(new IPCipherObfuscator(key, decrypt)); + return std::make_unique(key, decrypt); } uint32_t obf4(uint32_t orig) override diff --git a/pdns/dnswriter.cc b/pdns/dnswriter.cc index 5bf32a182e8a..0fd81b4d3d0b 100644 --- a/pdns/dnswriter.cc +++ b/pdns/dnswriter.cc @@ -382,6 +382,11 @@ template void GenericDNSPacketWriter::xfrBlob(co d_content.insert(d_content.end(), ptr, ptr+blob.size()); } +template void GenericDNSPacketWriter::xfrBlob(const std::vector& blob) +{ + d_content.insert(d_content.end(), blob.begin(), blob.end()); +} + template void GenericDNSPacketWriter::xfrBlobNoSpaces(const string& blob, int ) { xfrBlob(blob); diff --git a/pdns/dnswriter.hh b/pdns/dnswriter.hh index 55b5f6f0625d..485b5478337c 100644 --- a/pdns/dnswriter.hh +++ b/pdns/dnswriter.hh @@ -102,7 +102,7 @@ public: xfrBlob(val,16); } - void xfrCAWithoutPort(uint8_t version, ComboAddress &val) + void xfrCAWithoutPort(uint8_t version, const ComboAddress &val) { if (version == 4) xfrIP(val.sin4.sin_addr.s_addr); else if (version == 6) { @@ -113,7 +113,7 @@ public: else throw runtime_error("invalid IP protocol"); } - void xfrCAPort(ComboAddress &val) + void xfrCAPort(const ComboAddress &val) { uint16_t port; port = val.sin4.sin_port; @@ -131,6 +131,7 @@ public: void xfrText(const string& text, bool multi=false, bool lenField=true); void xfrUnquotedText(const string& text, bool lenField); void xfrBlob(const string& blob, int len=-1); + void xfrBlob(const vector& blob); void xfrSvcParamKeyVals(const set& kvs); void xfrBlobNoSpaces(const string& blob, int len=-1); void xfrHexBlob(const string& blob, bool keepReading=false); diff --git a/pdns/doh.hh b/pdns/doh.hh index a30a0ac6f924..16210d94933f 100644 --- a/pdns/doh.hh +++ b/pdns/doh.hh @@ -121,6 +121,11 @@ struct DOHFrontend return d_tlsConfig.d_ticketsKeyRotationDelay; } + bool isHTTPS() const + { + return !d_tlsConfig.d_certKeyPairs.empty(); + } + #ifndef HAVE_DNS_OVER_HTTPS void setup() { @@ -182,6 +187,7 @@ struct DOHUnit { DOHUnit() { + ids.ednsAdded = false; } DOHUnit(const DOHUnit&) = delete; DOHUnit& operator=(const DOHUnit&) = delete; @@ -204,21 +210,19 @@ struct DOHUnit void handleUDPResponse(PacketBuffer&& response, IDState&& state); - std::vector> headers; - PacketBuffer query; - PacketBuffer response; IDState ids; std::string sni; std::string path; std::string scheme; std::string host; - ComboAddress remote; - ComboAddress dest; + std::string contentType; + std::vector> headers; + PacketBuffer query; + PacketBuffer response; + std::shared_ptr downstream{nullptr}; st_h2o_req_t* req{nullptr}; DOHUnit** self{nullptr}; DOHServerConfig* dsc{nullptr}; - std::shared_ptr downstream{nullptr}; - std::string contentType; std::atomic d_refcnt{1}; size_t query_at{0}; size_t proxyProtocolPayloadSize{0}; @@ -231,7 +235,6 @@ struct DOHUnit the main DoH thread. */ uint16_t status_code{200}; - bool ednsAdded{false}; /* whether the query was re-sent to the backend over TCP after receiving a truncated answer over UDP */ bool tcp{false}; diff --git a/pdns/dynhandler.cc b/pdns/dynhandler.cc index a456464bcc69..31869cb1f805 100644 --- a/pdns/dynhandler.cc +++ b/pdns/dynhandler.cc @@ -323,7 +323,7 @@ string DLNotifyHandler(const vector&parts, Utility::pid_t ppid) if (parts[1] == "*") { vector domains; - B.getAllDomains(&domains); + B.getAllDomains(&domains, true, false); int total = 0; int notified = 0; @@ -380,7 +380,7 @@ string DLListZones(const vector&parts, Utility::pid_t ppid) UeberBackend B; g_log< domains; - B.getAllDomains(&domains); + B.getAllDomains(&domains, false, false); ostringstream ret; int kindFilter = -1; if (parts.size() > 1) { diff --git a/pdns/ednscookies.cc b/pdns/ednscookies.cc index 5e20819cefe6..5992b13f86db 100644 --- a/pdns/ednscookies.cc +++ b/pdns/ednscookies.cc @@ -19,33 +19,156 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif #include "ednscookies.hh" +#include "misc.hh" -bool getEDNSCookiesOptFromString(const string& option, EDNSCookiesOpt* eco) +#ifdef HAVE_CRYPTO_SHORTHASH +#include +#endif + +EDNSCookiesOpt::EDNSCookiesOpt(const std::string& option) { - return getEDNSCookiesOptFromString(option.c_str(), option.length(), eco); + getEDNSCookiesOptFromString(option.c_str(), option.length()); } -bool getEDNSCookiesOptFromString(const char* option, unsigned int len, EDNSCookiesOpt* eco) +EDNSCookiesOpt::EDNSCookiesOpt(const char* option, unsigned int len) { - if(len != 8 && len < 16) - return false; - eco->client = string(option, 8); - if (len > 8) { - eco->server = string(option + 8, len - 8); - } - return true; + getEDNSCookiesOptFromString(option, len); +} + +bool EDNSCookiesOpt::makeFromString(const std::string& option) +{ + getEDNSCookiesOptFromString(option.c_str(), option.length()); + return isWellFormed(); +} + +bool EDNSCookiesOpt::makeFromString(const char* option, unsigned int len) +{ + getEDNSCookiesOptFromString(option, len); + return isWellFormed(); } -string makeEDNSCookiesOptString(const EDNSCookiesOpt& eco) +string EDNSCookiesOpt::makeOptString() const { string ret; - if (eco.client.length() != 8) - return ret; - if (eco.server.length() != 0 && (eco.server.length() < 8 || eco.server.length() > 32)) + if (!isWellFormed()) return ret; - ret.assign(eco.client); - if (eco.server.length() != 0) - ret.append(eco.server); + ret.assign(client); + if (server.length() != 0) + ret.append(server); return ret; } + +void EDNSCookiesOpt::getEDNSCookiesOptFromString(const char* option, unsigned int len) +{ + client.clear(); + server.clear(); + if (len < 8) + return; + client = string(option, 8); + if (len > 8) { + server = string(option + 8, len - 8); + } +} + +bool EDNSCookiesOpt::isValid(const string& secret, const ComboAddress& source) const +{ +#ifdef HAVE_CRYPTO_SHORTHASH + if (server.length() != 16 || client.length() != 8) { + return false; + } + if (server[0] != '\x01') { + // Version is not 1, can't verify + return false; + } + uint32_t ts; + memcpy(&ts, &server[4], sizeof(ts)); + ts = ntohl(ts); + uint32_t now = static_cast(time(nullptr)); + // RFC 9018 section 4.3: + // The DNS server + // SHOULD allow cookies within a 1-hour period in the past and a + // 5-minute period into the future + if (rfc1982LessThan(now + 300, ts) && rfc1982LessThan(ts + 3600, now)) { + return false; + } + if (secret.length() != crypto_shorthash_KEYBYTES) { + return false; + } + + string toHash = client + server.substr(0, 8) + source.toByteString(); + string hashResult; + hashResult.resize(8); + crypto_shorthash( + reinterpret_cast(&hashResult[0]), + reinterpret_cast(&toHash[0]), + toHash.length(), + reinterpret_cast(&secret[0])); + return constantTimeStringEquals(server.substr(8), hashResult); +#else + return false; +#endif +} + +bool EDNSCookiesOpt::shouldRefresh() const +{ + if (server.size() < 16) { + return true; + } + uint32_t ts; + memcpy(&ts, &server[4], sizeof(ts)); + ts = ntohl(ts); + uint32_t now = static_cast(time(nullptr)); + // RFC 9018 section 4.3: + // The DNS server + // SHOULD allow cookies within a 1-hour period in the past and a + // 5-minute period into the future + // If this is not the case, we need to refresh + if (rfc1982LessThan(now + 300, ts) && rfc1982LessThan(ts + 3600, now)) { + return true; + } + + // RFC 9018 section 4.3: + // The DNS server SHOULD generate a new Server Cookie at least if the + // received Server Cookie from the client is more than half an hour old + return rfc1982LessThan(ts + 1800, now); +} + +bool EDNSCookiesOpt::makeServerCookie(const string& secret, const ComboAddress& source) +{ +#ifdef HAVE_CRYPTO_SHORTHASH + static_assert(EDNSCookieSecretSize == crypto_shorthash_KEYBYTES * 2, "The EDNSCookieSecretSize is not twice crypto_shorthash_KEYBYTES"); + + if (isValid(secret, source) && !shouldRefresh()) { + return true; + } + + if (secret.length() != crypto_shorthash_KEYBYTES) { + return false; + } + + server.clear(); + server.reserve(16); + server = "\x01"; // Version + server.resize(4, '\0'); // 3 reserved bytes + uint32_t now = htonl(static_cast(time(nullptr))); + server += string(reinterpret_cast(&now), sizeof(now)); + server.resize(8); + + string toHash = client; + toHash += server; + toHash += source.toByteString(); + server.resize(16); + crypto_shorthash( + reinterpret_cast(&server[8]), + reinterpret_cast(&toHash[0]), + toHash.length(), + reinterpret_cast(&secret[0])); + return true; +#else + return false; +#endif +} diff --git a/pdns/ednscookies.hh b/pdns/ednscookies.hh index 4e6f42e6cd62..7eff3c63fe99 100644 --- a/pdns/ednscookies.hh +++ b/pdns/ednscookies.hh @@ -21,13 +21,52 @@ */ #pragma once #include "namespaces.hh" +#include "iputils.hh" struct EDNSCookiesOpt { + static const size_t EDNSCookieSecretSize = 32; + static const size_t EDNSCookieOptSize = 24; + + EDNSCookiesOpt(){}; + EDNSCookiesOpt(const std::string& option); + EDNSCookiesOpt(const char* option, unsigned int len); + + bool makeFromString(const std::string& option); + bool makeFromString(const char* option, unsigned int len); + + size_t size() const + { + return server.size() + client.size(); + } + + bool isWellFormed() const + { + // RFC7873 section 5.2.2 + // In summary, valid cookie lengths are 8 and 16 to 40 inclusive. + return ( + client.size() == 8 && (server.size() == 0 || (server.size() >= 8 && server.size() <= 32))); + } + + bool isValid(const string& secret, const ComboAddress& source) const; + bool makeServerCookie(const string& secret, const ComboAddress& source); + string makeOptString() const; + string getServer() const + { + return server; + } + string getClient() const + { + return client; + } + +private: + bool shouldRefresh() const; + + // the client cookie string client; + // the server cookie string server; -}; -bool getEDNSCookiesOptFromString(const char* option, unsigned int len, EDNSCookiesOpt* eco); -bool getEDNSCookiesOptFromString(const string& option, EDNSCookiesOpt* eco); -string makeEDNSCookiesOptString(const EDNSCookiesOpt& eco); + void getEDNSCookiesOptFromString(const char* option, unsigned int len); +}; diff --git a/pdns/filterpo.cc b/pdns/filterpo.cc index 1716f9948125..2253d37215d6 100644 --- a/pdns/filterpo.cc +++ b/pdns/filterpo.cc @@ -50,13 +50,21 @@ bool DNSFilterEngine::Zone::findExactQNamePolicy(const DNSName& qname, DNSFilter bool DNSFilterEngine::Zone::findExactNSPolicy(const DNSName& qname, DNSFilterEngine::Policy& pol) const { - return findExactNamedPolicy(d_propolName, qname, pol); + if (findExactNamedPolicy(d_propolName, qname, pol)) { + pol.d_trigger = qname; + pol.d_trigger.appendRawLabel(rpzNSDnameName); + return true; + } + return false; } bool DNSFilterEngine::Zone::findNSIPPolicy(const ComboAddress& addr, DNSFilterEngine::Policy& pol) const { if (const auto fnd = d_propolNSAddr.lookup(addr)) { pol = fnd->second; + pol.d_trigger = Zone::maskToRPZ(fnd->first); + pol.d_trigger.appendRawLabel(rpzNSIPName); + pol.d_hit = addr.toString(); return true; } return false; @@ -66,6 +74,9 @@ bool DNSFilterEngine::Zone::findResponsePolicy(const ComboAddress& addr, DNSFilt { if (const auto fnd = d_postpolAddr.lookup(addr)) { pol = fnd->second; + pol.d_trigger = Zone::maskToRPZ(fnd->first); + pol.d_trigger.appendRawLabel(rpzIPName); + pol.d_hit = addr.toString(); return true; } return false; @@ -75,6 +86,9 @@ bool DNSFilterEngine::Zone::findClientPolicy(const ComboAddress& addr, DNSFilter { if (const auto fnd = d_qpolAddr.lookup(addr)) { pol = fnd->second; + pol.d_trigger = Zone::maskToRPZ(fnd->first); + pol.d_trigger.appendRawLabel(rpzClientIPName); + pol.d_hit = addr.toString(); return true; } return false; @@ -179,17 +193,13 @@ bool DNSFilterEngine::getProcessingPolicy(const DNSName& qname, const std::unord } if (z->findExactNSPolicy(qname, pol)) { // cerr<<"Had a hit on the nameserver ("<findExactNSPolicy(wc, pol)) { // cerr<<"Had a hit on the nameserver ("<findNSIPPolicy(address, pol)) { // cerr<<"Had a hit on the nameserver ("<findExactQNamePolicy(qname, pol)) { // cerr<<"Had a hit on the name of the query"<findExactQNamePolicy(wc, pol)) { // cerr<<"Had a hit on the name of the query"<findResponsePolicy(ca, pol)) { - pol.d_trigger = Zone::maskToRPZ(ca); - pol.d_trigger.appendRawLabel(rpzIPName); - pol.d_hit = ca.toString(); return true; } } diff --git a/pdns/fuzz_dnsdistcache.cc b/pdns/fuzz_dnsdistcache.cc index eb68950bddd1..c224449533e0 100644 --- a/pdns/fuzz_dnsdistcache.cc +++ b/pdns/fuzz_dnsdistcache.cc @@ -32,12 +32,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { /* dnsdist's version */ DNSDistPacketCache pcSkipCookies(10000); + // By default, cookies are not hashed pcSkipCookies.setECSParsingEnabled(true); - pcSkipCookies.setCookieHashing(false); DNSDistPacketCache pcHashCookies(10000); pcHashCookies.setECSParsingEnabled(true); - pcHashCookies.setCookieHashing(true); + // Do not skip cookies + pcHashCookies.setSkippedOptions({}); try { uint16_t qtype; diff --git a/pdns/fuzz_packetcache.cc b/pdns/fuzz_packetcache.cc index 98f99d372fbf..0c982d0e72ac 100644 --- a/pdns/fuzz_packetcache.cc +++ b/pdns/fuzz_packetcache.cc @@ -39,7 +39,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { try { static const std::unordered_set optionsToIgnore{ EDNSOptionCode::COOKIE }; - PacketCache::canHashPacket(input, false); + PacketCache::canHashPacket(input, optionsToIgnore); DNSName qname(input.data(), input.size(), sizeof(dnsheader), false); PacketCache::queryMatches(input, input, qname, optionsToIgnore); } @@ -52,7 +52,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { try { static const std::unordered_set optionsToIgnore{ EDNSOptionCode::COOKIE, EDNSOptionCode::ECS }; - PacketCache::canHashPacket(input, true); + PacketCache::canHashPacket(input, optionsToIgnore); DNSName qname(input.data(), input.size(), sizeof(dnsheader), false); PacketCache::queryMatches(input, input, qname, optionsToIgnore); } diff --git a/pdns/fuzz_zoneparsertng.cc b/pdns/fuzz_zoneparsertng.cc index 3285860a9d16..98e0d5ce383c 100644 --- a/pdns/fuzz_zoneparsertng.cc +++ b/pdns/fuzz_zoneparsertng.cc @@ -51,6 +51,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { ZoneParserTNG zpt(lines, g_rootdnsname); /* limit the number of steps for '$GENERATE' entries */ zpt.setMaxGenerateSteps(10000); + zpt.setMaxIncludes(20); DNSResourceRecord drr; while (zpt.get(drr)) { } diff --git a/pdns/histog.hh b/pdns/histog.hh index f96e4beab0fd..fb695283264f 100644 --- a/pdns/histog.hh +++ b/pdns/histog.hh @@ -20,9 +20,9 @@ struct LogHistogramBin }; template -std::vector createLogHistogram(const T& bins, - std::deque percentiles={0.001, 0.01, 0.1, 0.2, 0.5, 1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 94, 95, 96, 97, 97.5, 98, 98.5, 99, 99.5, 99.6, 99.9, 99.99, 99.999, 99.9999}) +std::vector createLogHistogram(const T& bins) { + std::deque percentiles={0.001, 0.01, 0.1, 0.2, 0.5, 1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 94, 95, 96, 97, 97.5, 98, 98.5, 99, 99.5, 99.6, 99.9, 99.99, 99.999, 99.9999}; uint64_t totcumul=0, sum=0; for(const auto& c: bins) { @@ -60,10 +60,9 @@ std::vector createLogHistogram(const T& bins, } template -void writeLogHistogramFile(const T& bins, std::ostream& out, std::deque percentiles={0.001, 0.01, 0.1, 0.2, 0.5, 1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 94, 95, 96, 97, 97.5, 98, 98.5, 99, 99.5, 99.6, 99.9, 99.99, 99.999, 99.9999} ) +void writeLogHistogramFile(const T& bins, std::ostream& out) { - - auto vec = createLogHistogram(bins, percentiles); + auto vec = createLogHistogram(bins); out<< R"(# set logscale xy # set mxtics 10 # set mytics 10 diff --git a/pdns/histogram.hh b/pdns/histogram.hh index beaaf795755f..7cfbcac343e2 100644 --- a/pdns/histogram.hh +++ b/pdns/histogram.hh @@ -22,12 +22,13 @@ #pragma once #include -#include #include #include #include #include +#include "stat_t.hh" + namespace pdns { @@ -50,7 +51,7 @@ struct AtomicBucket const std::string d_name; const uint64_t d_boundary{0}; - mutable std::atomic d_count{0}; + mutable stat_t d_count{0}; }; template @@ -178,6 +179,6 @@ private: using Histogram = BaseHistogram; -using AtomicHistogram = BaseHistogram>; +using AtomicHistogram = BaseHistogram; } // namespace pdns diff --git a/pdns/iputils.cc b/pdns/iputils.cc index c5aa2a87b235..05eed1245ba4 100644 --- a/pdns/iputils.cc +++ b/pdns/iputils.cc @@ -437,7 +437,7 @@ size_t sendMsgWithOptions(int fd, const char* buffer, size_t len, const ComboAdd return 0; } -template class NetmaskTree; +template class NetmaskTree; /* requires a non-blocking socket. On Linux, we could use MSG_DONTWAIT on a blocking socket diff --git a/pdns/iputils.hh b/pdns/iputils.hh index 32073637e1a1..b36f0e277236 100644 --- a/pdns/iputils.hh +++ b/pdns/iputils.hh @@ -30,7 +30,6 @@ #include #include "pdnsexception.hh" #include "misc.hh" -#include #include #include #include @@ -130,15 +129,15 @@ union ComboAddress { { uint32_t operator()(const ComboAddress& ca) const { - const unsigned char* start; - int len; - if(ca.sin4.sin_family == AF_INET) { - start =(const unsigned char*)&ca.sin4.sin_addr.s_addr; - len=4; + const unsigned char* start = nullptr; + uint32_t len = 0; + if (ca.sin4.sin_family == AF_INET) { + start = reinterpret_cast(&ca.sin4.sin_addr.s_addr); + len = 4; } else { - start =(const unsigned char*)&ca.sin6.sin6_addr.s6_addr; - len=16; + start = reinterpret_cast(&ca.sin6.sin6_addr.s6_addr); + len = 16; } return burtle(start, len, 0); } @@ -238,7 +237,7 @@ union ComboAddress { return false; int n=0; - const unsigned char*ptr = (unsigned char*) &sin6.sin6_addr.s6_addr; + const unsigned char* ptr = reinterpret_cast(&sin6.sin6_addr.s6_addr); for(n=0; n < 10; ++n) if(ptr[n]) return false; @@ -258,7 +257,7 @@ union ComboAddress { ret.sin4.sin_family=AF_INET; ret.sin4.sin_port=sin4.sin_port; - const unsigned char*ptr = (unsigned char*) &sin6.sin6_addr.s6_addr; + const unsigned char* ptr = reinterpret_cast(&sin6.sin6_addr.s6_addr); ptr+=(sizeof(sin6.sin6_addr.s6_addr) - sizeof(ret.sin4.sin_addr.s_addr)); memcpy(&ret.sin4.sin_addr.s_addr, ptr, sizeof(ret.sin4.sin_addr.s_addr)); return ret; @@ -268,7 +267,7 @@ union ComboAddress { { char host[1024]; int retval = 0; - if(sin4.sin_family && !(retval = getnameinfo((struct sockaddr*) this, getSocklen(), host, sizeof(host),0, 0, NI_NUMERICHOST))) + if(sin4.sin_family && !(retval = getnameinfo(reinterpret_cast(this), getSocklen(), host, sizeof(host),0, 0, NI_NUMERICHOST))) return string(host); else return "invalid "+string(gai_strerror(retval)); @@ -309,6 +308,14 @@ union ComboAddress { return toStringWithPortExcept(53); } + string toByteString() const + { + if (isIPv4()) { + return string(reinterpret_cast(&sin4.sin_addr.s_addr), sizeof(sin4.sin_addr.s_addr)); + } + return string(reinterpret_cast(&sin6.sin6_addr.s6_addr), sizeof(sin6.sin6_addr.s6_addr)); + } + void truncate(unsigned int bits) noexcept; uint16_t getPort() const @@ -364,7 +371,7 @@ union ComboAddress { index = 128 + index; } - uint8_t *ls_addr = (uint8_t*)sin6.sin6_addr.s6_addr; + const uint8_t* ls_addr = reinterpret_cast(sin6.sin6_addr.s6_addr); uint8_t byte_idx = index / 8; uint8_t bit_idx = index % 8; @@ -625,7 +632,7 @@ public: } //! Get the total number of address bits for this netmask (either 32 or 128 depending on IP version) - uint8_t getAddressBits() const + uint8_t getFullBits() const { return d_network.getBits(); } @@ -651,6 +658,7 @@ public: } return d_network.getBit(bit); } + private: ComboAddress d_network; uint32_t d_mask; @@ -678,12 +686,12 @@ private: * Please see NetmaskGroup for example of simple use case. Other usecases can be found * from GeoIPBackend and Sortlist, and from dnsdist. */ -template +template class NetmaskTree { public: class Iterator; - typedef Netmask key_type; + typedef K key_type; typedef T value_type; typedef std::pair node_type; typedef size_t size_type; @@ -697,9 +705,9 @@ private: explicit TreeNode() noexcept : parent(nullptr), node(), assigned(false), d_bits(0) { } - explicit TreeNode(const key_type& key) noexcept : + explicit TreeNode(const key_type& key) : parent(nullptr), node({key.getNormalized(), value_type()}), - assigned(false), d_bits(key.getAddressBits()) { + assigned(false), d_bits(key.getFullBits()) { } // addr_bits) + if (max_bits < 0 || max_bits > addr_bits) { max_bits = addr_bits; - - if (value.isIPv4()) - node = d_root->left.get(); - else if (value.isIPv6()) - node = d_root->right.get(); - else - throw NetmaskException("invalid address family"); - if (node == nullptr) return nullptr; - - node_type *ret = nullptr; - - int bits = 0; - for(; bits < max_bits; bits++) { - bool vall = value.getBit(-1-bits); - if (bits >= node->d_bits) { - // the end of the current node is reached; continue with the next - // (we keep track of last assigned node) - if (node->assigned && bits == node->node.first.getBits()) - ret = &node->node; - if (vall) { - if (!node->right) - break; - node = node->right.get(); - } else { - if (!node->left) - break; - node = node->left.get(); - } - continue; - } - if (bits >= node->node.first.getBits()) { - // the matching branch ends here - break; - } - bool valr = node->node.first.getBit(-1-bits); - if (vall != valr) { - // the branch matches just upto this point, yet continues in a different - // direction - break; - } } - // needed if we did not find one in loop - if (node->assigned && bits == node->node.first.getBits()) - ret = &node->node; - // this can be nullptr. - return ret; + return lookupImpl(key_type(value, max_bits), max_bits); } //left.get(); + else if (value.isIPv6()) + node = d_root->right.get(); + else + throw NetmaskException("invalid address family"); + if (node == nullptr) return nullptr; + + node_type *ret = nullptr; + + int bits = 0; + for(; bits < max_bits; bits++) { + bool vall = value.getBit(-1-bits); + if (bits >= node->d_bits) { + // the end of the current node is reached; continue with the next + // (we keep track of last assigned node) + if (node->assigned && bits == node->node.first.getBits()) + ret = &node->node; + if (vall) { + if (!node->right) + break; + node = node->right.get(); + } else { + if (!node->left) + break; + node = node->left.get(); + } + continue; + } + if (bits >= node->node.first.getBits()) { + // the matching branch ends here + break; + } + bool valr = node->node.first.getBit(-1-bits); + if (vall != valr) { + // the branch matches just upto this point, yet continues in a different + // direction + break; + } + } + // needed if we did not find one in loop + if (node->assigned && bits == node->node.first.getBits()) + ret = &node->node; + + // this can be nullptr. + return ret; + } + unique_ptr d_root; //> d_portMask); + port = port & mask; + } + + if (d_addrMask < d_addr.getBits()) { + if (d_portMask > 0) { + throw std::runtime_error("Trying to create a AddressAndPortRange with a reduced address mask (" + std::to_string(d_addrMask) + ") and a port range (" + std::to_string(d_portMask) + ")"); + } + d_addr = Netmask(d_addr, d_addrMask).getMaskedNetwork(); + } + d_addr.setPort(port); + } + + uint8_t getFullBits() const + { + return d_addr.getBits() + 16; + } + + uint8_t getBits() const + { + if (d_addrMask < d_addr.getBits()) { + return d_addrMask; + } + + return d_addr.getBits() + d_portMask; + } + + /** Get the value of the bit at the provided bit index. When the index >= 0, + the index is relative to the LSB starting at index zero. When the index < 0, + the index is relative to the MSB starting at index -1 and counting down. + */ + bool getBit(int index) const + { + if (index >= getFullBits()) { + return false; + } + if (index < 0) { + index = getFullBits() + index; + } + + if (index < 16) { + /* we are into the port bits */ + uint16_t port = d_addr.getPort(); + return ((port & (1U< rhs.d_addrMask) { + return true; + } + + if (d_addrMask < rhs.d_addrMask) { + return false; + } + + if (d_addr < rhs.d_addr) { + return true; + } + + if (d_addr > rhs.d_addr) { + return false; + } + + if (d_portMask > rhs.d_portMask) { + return true; + } + + if (d_portMask < rhs.d_portMask) { + return false; + } + + return d_addr.getPort() < rhs.d_addr.getPort(); + } + + bool operator>(const AddressAndPortRange& rhs) const + { + return rhs.operator<(*this); + } + + struct hash + { + uint32_t operator()(const AddressAndPortRange& apr) const + { + ComboAddress::addressOnlyHash hashOp; + uint16_t port = apr.d_addr.getPort(); + /* it's fine to hash the whole address and port because the non-relevant parts have + been masked to 0 */ + return burtle(reinterpret_cast(&port), sizeof(port), hashOp(apr.d_addr)); + } + }; + +private: + ComboAddress d_addr; + uint8_t d_addrMask; + /* only used for v4 addresses */ + uint8_t d_portMask; +}; + int SSocket(int family, int type, int flags); int SConnect(int sockfd, const ComboAddress& remote); /* tries to connect to remote for a maximum of timeout seconds. @@ -1455,4 +1649,3 @@ bool isTCPSocketUsable(int sock); extern template class NetmaskTree; ComboAddress parseIPAndPort(const std::string& input, uint16_t port); - diff --git a/pdns/ixfr.cc b/pdns/ixfr.cc index 9b1b4b33d34d..1154eb01366c 100644 --- a/pdns/ixfr.cc +++ b/pdns/ixfr.cc @@ -116,7 +116,7 @@ vector, vector > > processIXFRRecords(const Co } } - ret.push_back(make_pair(remove,add)); + ret.emplace_back(remove, add); } return ret; @@ -174,13 +174,21 @@ vector, vector > > getIXFRDeltas(const ComboAd std::shared_ptr primarySOA = nullptr; vector records; size_t receivedBytes = 0; - int8_t ixfrInProgress = -2; std::string reply; + enum transferStyle { Unknown, AXFR, IXFR } style = Unknown; + const unsigned int expectedSOAForAXFR = 2; + const unsigned int expectedSOAForIXFR = 3; + unsigned int primarySOACount = 0; + for(;;) { - // IXFR end - if (ixfrInProgress >= 0) + // IXFR or AXFR style end reached? We don't want to process trailing data after the closing SOA + if (style == AXFR && primarySOACount == expectedSOAForAXFR) { + break; + } + else if (style == IXFR && primarySOACount == expectedSOAForIXFR) { break; + } if(s.read((char*)&len, sizeof(len)) != sizeof(len)) break; @@ -225,16 +233,31 @@ vector, vector > > getIXFRDeltas(const ComboAd return ret; } primarySOA = sr; + ++primarySOACount; } else if (r.first.d_type == QType::SOA) { auto sr = getRR(r.first); if (!sr) { throw std::runtime_error("Error getting the content of SOA record of IXFR answer for zone '"+zone.toLogString()+"' from primary '"+primary.toStringWithPort()+"'"); } - // we hit the last SOA record - // IXFR is considered to be done if we hit the last SOA record twice + // we hit a marker SOA record if (primarySOA->d_st.serial == sr->d_st.serial) { - ixfrInProgress++; + ++primarySOACount; + } + } + // When we see the 2nd record, we can decide what the style is + if (records.size() == 1 && style == Unknown) { + if (r.first.d_type != QType::SOA) { + // Non-empty AXFR style has a non-SOA record following the first SOA + style = AXFR; + } + else if (primarySOACount == expectedSOAForAXFR) { + // Empty zone AXFR style: start SOA is immediately followed by end marker SOA + style = AXFR; + } + else { + // IXFR has a 2nd SOA (with different serial) following the first + style = IXFR; } } @@ -253,7 +276,21 @@ vector, vector > > getIXFRDeltas(const ComboAd } } - // cout<<"Got "<(new WebServer(listenAddress.toString(), listenAddress.getPort()))) +IXFRDistWebServer::IXFRDistWebServer(const ComboAddress& listenAddress, const NetmaskGroup& acl, const string& loglevel) : + d_ws(std::make_unique(listenAddress.toString(), listenAddress.getPort())) { d_ws->setACL(acl); d_ws->setLogLevel(loglevel); diff --git a/pdns/ixfrdist.cc b/pdns/ixfrdist.cc index f7d58eb30fd2..aea08ee066fb 100644 --- a/pdns/ixfrdist.cc +++ b/pdns/ixfrdist.cc @@ -42,6 +42,7 @@ #include "mplexer.hh" #include "misc.hh" #include "iputils.hh" +#include "lock.hh" #include "logger.hh" #include "ixfrdist-stats.hh" #include "ixfrdist-web.hh" @@ -147,8 +148,7 @@ struct ixfrdistdomain_t { static map g_domainConfigs; // Map domains and their data -static std::map> g_soas; -static std::mutex g_soas_mutex; +static LockGuarded>> g_soas; // Condition variable for TCP handling static std::condition_variable g_tcpHandlerCV; @@ -218,7 +218,7 @@ static void cleanUpDomain(const DNSName& domain, const uint16_t& keep, const str // And delete all the old ones { // Lock to ensure no one reads this. - std::lock_guard guard(g_soas_mutex); + auto lock = g_soas.lock(); for (auto iter = zoneVersions.cbegin(); iter != zoneVersions.cend() - keep; ++iter) { string fname = dir + "/" + std::to_string(*iter); g_log< getCurrentZoneInfo(const DNSName& domain) { - std::lock_guard guard(g_soas_mutex); - return g_soas[domain]; + return (*g_soas.lock())[domain]; } static void updateCurrentZoneInfo(const DNSName& domain, std::shared_ptr& newInfo) { - std::lock_guard guard(g_soas_mutex); - g_soas[domain] = newInfo; + auto soas = g_soas.lock(); + (*soas)[domain] = newInfo; g_stats.setSOASerial(domain, newInfo->soa->d_st.serial); // FIXME: also report zone size? } diff --git a/pdns/libssl.cc b/pdns/libssl.cc index b667d27d01cf..2c0e8be68251 100644 --- a/pdns/libssl.cc +++ b/pdns/libssl.cc @@ -15,11 +15,15 @@ #include #include #include +#include #ifdef HAVE_LIBSODIUM #include #endif /* HAVE_LIBSODIUM */ +#undef CERT +#include "misc.hh" + #if (OPENSSL_VERSION_NUMBER < 0x1010000fL || (defined LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x2090100fL) /* OpenSSL < 1.1.0 needs support for threading/locking in the calling application. */ @@ -779,9 +783,15 @@ static void libssl_key_log_file_callback(const SSL* ssl, const char* line) std::unique_ptr libssl_set_key_log_file(std::unique_ptr& ctx, const std::string& logFile) { #ifdef HAVE_SSL_CTX_SET_KEYLOG_CALLBACK - auto fp = std::unique_ptr(fopen(logFile.c_str(), "a"), fclose); + int fd = open(logFile.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0600); + if (fd == -1) { + unixDie("Error opening TLS log file '" + logFile + "'"); + } + auto fp = std::unique_ptr(fdopen(fd, "a"), fclose); if (!fp) { - throw std::runtime_error("Error opening TLS log file '" + logFile + "'"); + int error = errno; // close might clobber errno + close(fd); + throw std::runtime_error("Error opening TLS log file '" + logFile + "': " + stringerror(error)); } SSL_CTX_set_ex_data(ctx.get(), s_keyLogIndex, fp.get()); @@ -793,6 +803,40 @@ std::unique_ptr libssl_set_key_log_file(std::unique_ptr>& protos) +{ +#ifdef HAVE_SSL_CTX_SET_ALPN_PROTOS + std::vector wire; + for (const auto& proto : protos) { + if (proto.size() > std::numeric_limits::max()) { + throw std::runtime_error("Invalid ALPN value"); + } + uint8_t length = proto.size(); + wire.push_back(length); + wire.insert(wire.end(), proto.begin(), proto.end()); + } + return SSL_CTX_set_alpn_protos(ctx, wire.data(), wire.size()) == 0; +#else + return false; +#endif +} + + std::string libssl_get_error_string() { BIO *mem = BIO_new(BIO_s_mem()); diff --git a/pdns/libssl.hh b/pdns/libssl.hh index b090afa7b146..4561b4a496f1 100644 --- a/pdns/libssl.hh +++ b/pdns/libssl.hh @@ -126,6 +126,13 @@ std::unique_ptr libssl_init_server_context(const TLS std::unique_ptr libssl_set_key_log_file(std::unique_ptr& ctx, const std::string& logFile); +/* called in a client context, if the client advertised more than one ALPN values and the server returned more than one as well, to select the one to use. */ +void libssl_set_npn_select_callback(SSL_CTX* ctx, int (*cb)(SSL* s, unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen, void* arg), void* arg); +/* called in a server context, to select an ALPN value advertised by the client if any */ +void libssl_set_alpn_select_callback(SSL_CTX* ctx, int (*cb)(SSL* s, const unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen, void* arg), void* arg); +/* set the supported ALPN protos in client context */ +bool libssl_set_alpn_protos(SSL_CTX* ctx, const std::vector>& protos); + std::string libssl_get_error_string(); #endif /* HAVE_LIBSSL */ diff --git a/pdns/lock.hh b/pdns/lock.hh index 1b60d46a5e31..e8bd82988da8 100644 --- a/pdns/lock.hh +++ b/pdns/lock.hh @@ -110,9 +110,8 @@ public: ReadLock(const ReadLock& rhs) = delete; ReadLock& operator=(const ReadLock& rhs) = delete; - ReadLock(ReadLock&& rhs) + ReadLock(ReadLock&& rhs) : d_lock(std::move(rhs.d_lock)) { - d_lock = std::move(rhs.d_lock); } private: @@ -136,9 +135,8 @@ public: WriteLock(const WriteLock& rhs) = delete; WriteLock& operator=(const WriteLock& rhs) = delete; - WriteLock(WriteLock&& rhs) + WriteLock(WriteLock&& rhs) : d_lock(std::move(rhs.d_lock)) { - d_lock = std::move(rhs.d_lock); } private: @@ -290,6 +288,11 @@ public: return LockGuardedHolder(d_value, d_mutex); } + LockGuardedHolder read_only_lock() const + { + return LockGuardedHolder(d_value, d_mutex); + } + private: std::mutex d_mutex; T d_value; diff --git a/pdns/lua-auth4.cc b/pdns/lua-auth4.cc index 2045d9e1b5df..6707e60f1a73 100644 --- a/pdns/lua-auth4.cc +++ b/pdns/lua-auth4.cc @@ -10,7 +10,6 @@ #include #include "sstuff.hh" #include -#include #include "ueberbackend.hh" @@ -41,10 +40,10 @@ void AuthLua4::postPrepareContext() { d_lw->registerFunction("parse", [](DNSPacket &p, const char *mesg, size_t len){ return p.parse(mesg, len); }); d_lw->registerFunction("getString", [](DNSPacket &p) { return p.getString(); }); d_lw->registerFunction("setRemote", [](DNSPacket &p, const ComboAddress &ca) { p.setRemote(&ca); }); - d_lw->registerFunction("getRemote", [](DNSPacket &p) { return p.getRemote(); }); + d_lw->registerFunction("getRemote", [](DNSPacket &p) { return p.getInnerRemote(); }); d_lw->registerFunction("getRealRemote", [](DNSPacket &p) { return p.getRealRemote(); }); d_lw->registerFunction("getLocal", [](DNSPacket &p) { return p.getLocal(); }); - d_lw->registerFunction("getRemotePort", [](DNSPacket &p) { return p.getRemotePort(); }); + d_lw->registerFunction("getRemotePort", [](DNSPacket &p) { return p.getInnerRemote().getPort(); }); d_lw->registerFunction()>("getQuestion", [](DNSPacket &p) { return std::make_tuple(p.qdomain.toString(), static_cast(p.qtype.getCode())); }); d_lw->registerFunction("setA", [](DNSPacket &p, bool a) { return p.setA(a); }); d_lw->registerFunction("setID", [](DNSPacket &p, unsigned int id) { return p.setID(static_cast(id)); }); @@ -153,7 +152,7 @@ bool AuthLua4::updatePolicy(const DNSName &qname, const QType& qtype, const DNSN upq.qtype = qtype.getCode(); upq.zonename = zonename; upq.local = packet.getLocal(); - upq.remote = packet.getRemote(); + upq.remote = packet.getInnerRemote(); upq.realRemote = packet.getRealRemote(); upq.tsigName = packet.getTSIGKeyname(); upq.peerPrincipal = packet.d_peer_principal; diff --git a/pdns/lua-base4.cc b/pdns/lua-base4.cc index efbf71257a27..31c2da0cf730 100644 --- a/pdns/lua-base4.cc +++ b/pdns/lua-base4.cc @@ -37,7 +37,7 @@ void BaseLua4::loadString(const std::string &script) { void BaseLua4::getFeatures(Features &) { } void BaseLua4::prepareContext() { - d_lw = std::unique_ptr(new LuaContext); + d_lw = std::make_unique(); // lua features available Features features; @@ -244,7 +244,8 @@ void BaseLua4::prepareContext() { {"YXRRSET", RCode::YXRRSet }, {"NXRRSET", RCode::NXRRSet }, {"NOTAUTH", RCode::NotAuth }, - {"NOTZONE", RCode::NotZone }}; + {"NOTZONE", RCode::NotZone }, + {"DROP", -2 }}; // To give backport-incompatibility warning for(const auto& rcode : rcodes) d_pd.push_back({rcode.first, rcode.second}); diff --git a/pdns/lua-record.cc b/pdns/lua-record.cc index 49dd69fd4ee3..dfa118a6fbaf 100644 --- a/pdns/lua-record.cc +++ b/pdns/lua-record.cc @@ -1,10 +1,10 @@ #include #include -#include #include #include #include "version.hh" #include "ext/luawrapper/include/LuaContext.hpp" +#include "lock.hh" #include "lua-auth4.hh" #include "sstuff.hh" #include "minicurl.hh" @@ -73,7 +73,6 @@ class IsUpOracle std::atomic lastAccess{0}; }; - ReadWriteLock d_lock; public: IsUpOracle() { @@ -158,8 +157,9 @@ class IsUpOracle std::vector> results; std::vector toDelete; { - ReadLock lock{&d_lock}; // make sure there's no insertion - for (auto& it: d_statuses) { + // make sure there's no insertion + auto statuses = d_statuses.read_lock(); + for (auto& it: *statuses) { auto& desc = it.first; auto& state = it.second; @@ -178,9 +178,9 @@ class IsUpOracle future.wait(); } if (!toDelete.empty()) { - WriteLock lock{&d_lock}; + auto statuses = d_statuses.write_lock(); for (auto& it: toDelete) { - d_statuses.erase(it); + statuses->erase(it); } } std::this_thread::sleep_until(checkStart + std::chrono::seconds(g_luaHealthChecksInterval)); @@ -188,14 +188,14 @@ class IsUpOracle } typedef map> statuses_t; - statuses_t d_statuses; + SharedLockGuarded d_statuses; std::unique_ptr d_checkerThread; void setStatus(const CheckDesc& cd, bool status) { - ReadLock lock{&d_lock}; - auto& state = d_statuses[cd]; + auto statuses = d_statuses.write_lock(); + auto& state = (*statuses)[cd]; state->status = status; if (state->first) { state->first = false; @@ -229,13 +229,13 @@ class IsUpOracle bool IsUpOracle::isUp(const CheckDesc& cd) { if (!d_checkerThread) { - d_checkerThread = std::unique_ptr(new std::thread(&IsUpOracle::checkThread, this)); + d_checkerThread = std::make_unique([this] { return checkThread(); }); } time_t now = time(nullptr); { - ReadLock lock{&d_lock}; - auto iter = d_statuses.find(cd); - if (iter != d_statuses.end()) { + auto statuses = d_statuses.read_lock(); + auto iter = statuses->find(cd); + if (iter != statuses->end()) { iter->second->lastAccess = now; return iter->second->status; } @@ -245,10 +245,10 @@ bool IsUpOracle::isUp(const CheckDesc& cd) ComboAddress src(cd.opts.at("source")); } { - WriteLock lock{&d_lock}; + auto statuses = d_statuses.write_lock(); // Make sure we don't insert new entry twice now we have the lock - if (d_statuses.find(cd) == d_statuses.end()) { - d_statuses[cd] = std::unique_ptr(new CheckState{now}); + if (statuses->find(cd) == statuses->end()) { + (*statuses)[cd] = std::make_unique(now); } } return false; @@ -326,7 +326,7 @@ static ComboAddress pickwrandom(const vector >& wips) vector > pick; for(auto& i : wips) { sum += i.first; - pick.push_back({sum, i.second}); + pick.emplace_back(sum, i.second); } int r = dns_random(sum); auto p = upper_bound(pick.begin(), pick.end(), r, [](int rarg, const decltype(pick)::value_type& a) { return rarg < a.first; }); @@ -443,14 +443,16 @@ static ComboAddress pickclosest(const ComboAddress& bestwho, const vector lookup(const DNSName& name, uint16_t qtype, int zoneid) { - static UeberBackend ub; - static std::mutex mut; - std::lock_guard lock(mut); - ub.lookup(QType(qtype), name, zoneid); + static LockGuarded s_ub; + DNSZoneRecord dr; vector ret; - while(ub.get(dr)) { - ret.push_back(dr); + { + auto ub = s_ub.lock(); + ub->lookup(QType(qtype), name, zoneid); + while (ub->get(dr)) { + ret.push_back(dr); + } } return ret; } @@ -970,7 +972,7 @@ std::vector> luaSynth(const std::string& code, cons LuaContext& lua = *s_LUA->getLua(); - s_lua_record_ctx = std::unique_ptr(new lua_record_ctx_t()); + s_lua_record_ctx = std::make_unique(); s_lua_record_ctx->qname = query; s_lua_record_ctx->zone = zone; s_lua_record_ctx->zoneid = zoneid; @@ -978,7 +980,7 @@ std::vector> luaSynth(const std::string& code, cons lua.writeVariable("qname", query); lua.writeVariable("zone", zone); lua.writeVariable("zoneid", zoneid); - lua.writeVariable("who", dnsp.getRemote()); + lua.writeVariable("who", dnsp.getInnerRemote()); lua.writeVariable("dh", (dnsheader*)&dnsp.d); lua.writeVariable("dnssecOK", dnsp.d_dnssecOk); lua.writeVariable("tcp", dnsp.d_tcp); @@ -989,7 +991,7 @@ std::vector> luaSynth(const std::string& code, cons } else { lua.writeVariable("ecswho", nullptr); - s_lua_record_ctx->bestwho = dnsp.getRemote(); + s_lua_record_ctx->bestwho = dnsp.getInnerRemote(); } lua.writeVariable("bestwho", s_lua_record_ctx->bestwho); diff --git a/pdns/lua-recursor4.cc b/pdns/lua-recursor4.cc index 98244c9da6c9..c140e9e0f30e 100644 --- a/pdns/lua-recursor4.cc +++ b/pdns/lua-recursor4.cc @@ -490,9 +490,18 @@ void RecursorLua4::getFeatures(Features& features) { // Add key-values pairs below. // Make sure you add string values explicitly converted to string. - // e.g. features.push_back(make_pair("somekey", string("stringvalue")); + // e.g. features.emplace_back("somekey", string("stringvalue"); // Both int and double end up as a lua number type. - features.push_back(make_pair("PR8001_devicename", true)); + features.emplace_back("PR8001_devicename", true); +} + +static void warnDrop(const RecursorLua4::DNSQuestion& dq) +{ + if (dq.rcode == -2) { + g_log << Logger::Error << "Returning -2 (pdns.DROP) is not supported anymore, see https://docs.powerdns.com/recursor/lua-scripting/hooks.html#hooksemantics" << endl; + // We *could* set policy here, but that would also mean interfering with rcode and the return code of the hook. + // So leave it at the error message. + } } void RecursorLua4::maintenance() const @@ -502,48 +511,93 @@ void RecursorLua4::maintenance() const } } -bool RecursorLua4::prerpz(DNSQuestion& dq, int& ret) const +bool RecursorLua4::prerpz(DNSQuestion& dq, int& ret, RecEventTrace& et) const { - return genhook(d_prerpz, dq, ret); + if (!d_prerpz) { + return false; + } + et.add(RecEventTrace::LuaPreRPZ); + bool ok = genhook(d_prerpz, dq, ret); + et.add(RecEventTrace::LuaPreRPZ, ok, false); + warnDrop(dq); + return ok; } -bool RecursorLua4::preresolve(DNSQuestion& dq, int& ret) const +bool RecursorLua4::preresolve(DNSQuestion& dq, int& ret, RecEventTrace& et) const { - return genhook(d_preresolve, dq, ret); + if (!d_preresolve) { + return false; + } + et.add(RecEventTrace::LuaPreResolve); + bool ok = genhook(d_preresolve, dq, ret); + et.add(RecEventTrace::LuaPreResolve, ok, false); + warnDrop(dq); + return ok; } -bool RecursorLua4::nxdomain(DNSQuestion& dq, int& ret) const +bool RecursorLua4::nxdomain(DNSQuestion& dq, int& ret, RecEventTrace& et) const { - return genhook(d_nxdomain, dq, ret); + if (!d_nxdomain) { + return false; + } + et.add(RecEventTrace::LuaNXDomain); + bool ok = genhook(d_nxdomain, dq, ret); + et.add(RecEventTrace::LuaNXDomain, ok, false); + warnDrop(dq); + return ok; } -bool RecursorLua4::nodata(DNSQuestion& dq, int& ret) const +bool RecursorLua4::nodata(DNSQuestion& dq, int& ret, RecEventTrace& et) const { - return genhook(d_nodata, dq, ret); + if (!d_nodata) { + return false; + } + et.add(RecEventTrace::LuaNoData); + bool ok = genhook(d_nodata, dq, ret); + et.add(RecEventTrace::LuaNoData, ok, false); + warnDrop(dq); + return ok; } -bool RecursorLua4::postresolve(DNSQuestion& dq, int& ret) const +bool RecursorLua4::postresolve(DNSQuestion& dq, int& ret, RecEventTrace& et) const { - return genhook(d_postresolve, dq, ret); + if (!d_postresolve) { + return false; + } + et.add(RecEventTrace::LuaPostResolve); + bool ok = genhook(d_postresolve, dq, ret); + et.add(RecEventTrace::LuaPostResolve, ok, false); + warnDrop(dq); + return ok; } -bool RecursorLua4::preoutquery(const ComboAddress& ns, const ComboAddress& requestor, const DNSName& query, const QType& qtype, bool isTcp, vector& res, int& ret) const +bool RecursorLua4::preoutquery(const ComboAddress& ns, const ComboAddress& requestor, const DNSName& query, const QType& qtype, bool isTcp, vector& res, int& ret, RecEventTrace& et) const { + if (!d_preoutquery) { + return false; + } bool variableAnswer = false; bool wantsRPZ = false; bool logQuery = false; bool addPaddingToResponse = false; RecursorLua4::DNSQuestion dq(ns, requestor, query, qtype.getCode(), isTcp, variableAnswer, wantsRPZ, logQuery, addPaddingToResponse); dq.currentRecords = &res; - - return genhook(d_preoutquery, dq, ret); + et.add(RecEventTrace::LuaPreOutQuery); + bool ok = genhook(d_preoutquery, dq, ret); + et.add(RecEventTrace::LuaPreOutQuery, ok, false); + warnDrop(dq); + return ok; } -bool RecursorLua4::ipfilter(const ComboAddress& remote, const ComboAddress& local, const struct dnsheader& dh) const +bool RecursorLua4::ipfilter(const ComboAddress& remote, const ComboAddress& local, const struct dnsheader& dh, RecEventTrace& et) const { - if (d_ipfilter) - return d_ipfilter(remote, local, dh); - return false; // don't block + if (!d_ipfilter) { + return false; // Do not block + } + et.add(RecEventTrace::LuaIPFilter); + bool ok = d_ipfilter(remote, local, dh); + et.add(RecEventTrace::LuaIPFilter, ok, false); + return ok; } bool RecursorLua4::policyHitEventFilter(const ComboAddress& remote, const DNSName& qname, const QType& qtype, bool tcp, DNSFilterEngine::Policy& policy, std::unordered_set& tags, std::unordered_map& discardedPolicies) const @@ -711,7 +765,7 @@ RecursorLua4::~RecursorLua4() {} const char* pdns_ffi_param_get_qname(pdns_ffi_param_t* ref) { if (!ref->qnameStr) { - ref->qnameStr = std::unique_ptr(new std::string(ref->params.qname.toStringNoDot())); + ref->qnameStr = std::make_unique(ref->params.qname.toStringNoDot()); } return ref->qnameStr->c_str(); @@ -732,7 +786,7 @@ uint16_t pdns_ffi_param_get_qtype(const pdns_ffi_param_t* ref) const char* pdns_ffi_param_get_remote(pdns_ffi_param_t* ref) { if (!ref->remoteStr) { - ref->remoteStr = std::unique_ptr(new std::string(ref->params.remote.toString())); + ref->remoteStr = std::make_unique(ref->params.remote.toString()); } return ref->remoteStr->c_str(); @@ -763,7 +817,7 @@ uint16_t pdns_ffi_param_get_remote_port(const pdns_ffi_param_t* ref) const char* pdns_ffi_param_get_local(pdns_ffi_param_t* ref) { if (!ref->localStr) { - ref->localStr = std::unique_ptr(new std::string(ref->params.local.toString())); + ref->localStr = std::make_unique(ref->params.local.toString()); } return ref->localStr->c_str(); @@ -786,7 +840,7 @@ const char* pdns_ffi_param_get_edns_cs(pdns_ffi_param_t* ref) } if (!ref->ednssubnetStr) { - ref->ednssubnetStr = std::unique_ptr(new std::string(ref->params.ednssubnet.toStringNoMask())); + ref->ednssubnetStr = std::make_unique(ref->params.ednssubnet.toStringNoMask()); } return ref->ednssubnetStr->c_str(); diff --git a/pdns/lua-recursor4.hh b/pdns/lua-recursor4.hh index a1ff21e89d86..a86c9eaa8e8e 100644 --- a/pdns/lua-recursor4.hh +++ b/pdns/lua-recursor4.hh @@ -35,6 +35,7 @@ #include "lua-base4.hh" #include "proxy-protocol.hh" #include "noinitvector.hh" +#include "rec-eventtrace.hh" #include @@ -185,14 +186,14 @@ public: unsigned int gettag_ffi(FFIParams&) const; void maintenance() const; - bool prerpz(DNSQuestion& dq, int& ret) const; - bool preresolve(DNSQuestion& dq, int& ret) const; - bool nxdomain(DNSQuestion& dq, int& ret) const; - bool nodata(DNSQuestion& dq, int& ret) const; - bool postresolve(DNSQuestion& dq, int& ret) const; - - bool preoutquery(const ComboAddress& ns, const ComboAddress& requestor, const DNSName& query, const QType& qtype, bool isTcp, vector& res, int& ret) const; - bool ipfilter(const ComboAddress& remote, const ComboAddress& local, const struct dnsheader&) const; + bool prerpz(DNSQuestion& dq, int& ret, RecEventTrace&) const; + bool preresolve(DNSQuestion& dq, int& ret, RecEventTrace&) const; + bool nxdomain(DNSQuestion& dq, int& ret, RecEventTrace&) const; + bool nodata(DNSQuestion& dq, int& ret, RecEventTrace&) const; + bool postresolve(DNSQuestion& dq, int& ret, RecEventTrace&) const; + + bool preoutquery(const ComboAddress& ns, const ComboAddress& requestor, const DNSName& query, const QType& qtype, bool isTcp, vector& res, int& ret, RecEventTrace& et) const; + bool ipfilter(const ComboAddress& remote, const ComboAddress& local, const struct dnsheader&, RecEventTrace&) const; bool policyHitEventFilter(const ComboAddress& remote, const DNSName& qname, const QType& qtype, bool tcp, DNSFilterEngine::Policy& policy, std::unordered_set& tags, std::unordered_map& discardedPolicies) const; diff --git a/pdns/lwres.cc b/pdns/lwres.cc index 1441a39edbaf..90f1c84d4d49 100644 --- a/pdns/lwres.cc +++ b/pdns/lwres.cc @@ -52,12 +52,14 @@ #include "rec-protozero.hh" #include "uuid-utils.hh" +#include "rec-tcpout.hh" + +thread_local TCPOutConnectionManager t_tcp_manager; #ifdef HAVE_FSTRM #include "dnstap.hh" #include "fstrm_logger.hh" - bool g_syslog; static bool isEnabledForQueries(const std::shared_ptr>>& fstreamLoggers) @@ -119,7 +121,7 @@ static void logFstreamResponse(const std::shared_ptr>>& outgoingLoggers, boost::optional initialRequestId, const boost::uuids::uuid& uuid, const ComboAddress& ip, const DNSName& domain, int type, uint16_t qid, bool doTCP, size_t bytes, boost::optional& srcmask) +static void logOutgoingQuery(const std::shared_ptr>>& outgoingLoggers, boost::optional initialRequestId, const boost::uuids::uuid& uuid, const ComboAddress& ip, const DNSName& domain, int type, uint16_t qid, bool doTCP, bool tls, size_t bytes, boost::optional& srcmask) { if (!outgoingLoggers) { return; @@ -143,7 +145,16 @@ static void logOutgoingQuery(const std::shared_ptr>>& outgoingLoggers, boost::optional initialRequestId, const boost::uuids::uuid& uuid, const ComboAddress& ip, const DNSName& domain, int type, uint16_t qid, bool doTCP, boost::optional& srcmask, size_t bytes, int rcode, const std::vector& records, const struct timeval& queryTime, const std::set& exportTypes) +static void logIncomingResponse(const std::shared_ptr>>& outgoingLoggers, boost::optional initialRequestId, const boost::uuids::uuid& uuid, const ComboAddress& ip, const DNSName& domain, int type, uint16_t qid, bool doTCP, bool tls, boost::optional& srcmask, size_t bytes, int rcode, const std::vector& records, const struct timeval& queryTime, const std::set& exportTypes) { if (!outgoingLoggers) { return; @@ -191,7 +202,15 @@ static void logIncomingResponse(const std::shared_ptr(g_networkTimeoutMsec) % 1000 * 1000}; + Socket s(ip.sin4.sin_family, SOCK_STREAM); + s.setNonBlocking(); + ComboAddress localip = pdns::getQueryLocalAddress(ip.sin4.sin_family, 0); + s.bind(localip); + + std::shared_ptr tlsCtx{nullptr}; + if (dnsOverTLS) { + TLSContextParameters tlsParams; + tlsParams.d_provider = "openssl"; + tlsParams.d_validateCertificates = false; + // tlsParams.d_caStore + tlsCtx = getTLSContext(tlsParams); + if (tlsCtx == nullptr) { + g_log << Logger::Error << "DoT to " << ip << " requested but not available" << endl; + dnsOverTLS = false; + } + } + connection.d_handler = std::make_shared("", s.releaseHandle(), timeout, tlsCtx, now.tv_sec); + // Returned state ignored + // This can throw an exception, retry will need to happen at higher level + connection.d_handler->tryConnect(SyncRes::s_tcp_fast_open_connect, ip); + return true; +} + +static LWResult::Result tcpsendrecv(const ComboAddress& ip, TCPOutConnectionManager::Connection& connection, + ComboAddress& localip, const vector& vpacket, size_t& len, PacketBuffer& buf) +{ + socklen_t slen = ip.getSocklen(); + uint16_t tlen = htons(vpacket.size()); + const char *lenP = reinterpret_cast(&tlen); + + localip.sin4.sin_family = ip.sin4.sin_family; + getsockname(connection.d_handler->getDescriptor(), reinterpret_cast(&localip), &slen); + + PacketBuffer packet; + packet.reserve(2 + vpacket.size()); + packet.insert(packet.end(), lenP, lenP + 2); + packet.insert(packet.end(), vpacket.begin(), vpacket.end()); + + LWResult::Result ret = asendtcp(packet, connection.d_handler); + if (ret != LWResult::Result::Success) { + return ret; + } + + ret = arecvtcp(packet, 2, connection.d_handler, false); + if (ret != LWResult::Result::Success) { + return ret; + } + + memcpy(&tlen, packet.data(), sizeof(tlen)); + len = ntohs(tlen); // switch to the 'len' shared with the rest of the calling function + + // XXX receive into buf directly? + packet.resize(len); + ret = arecvtcp(packet, len, connection.d_handler, false); + if (ret != LWResult::Result::Success) { + return ret; + } + buf.resize(len); + memcpy(buf.data(), packet.data(), len); + return LWResult::Result::Success; +} + /** lwr is only filled out in case 1 was returned, and even when returning 1 for 'success', lwr might contain DNS errors Never throws! */ -LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, const std::shared_ptr>>& outgoingLoggers, const std::shared_ptr>>& fstrmLoggers, const std::set& exportTypes, LWResult *lwr, bool* chained) +static LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, const std::shared_ptr>>& outgoingLoggers, const std::shared_ptr>>& fstrmLoggers, const std::set& exportTypes, LWResult *lwr, bool* chained, TCPOutConnectionManager::Connection& connection) { size_t len; size_t bufsize=g_outgoingEDNSBufsize; @@ -269,7 +361,7 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int outgoingECSBits = srcmask->getBits(); outgoingECSAddr = srcmask->getNetwork(); // cout<<"Adding request mask: "<d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, vpacket.size(), srcmask); + logOutgoingQuery(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, dnsOverTLS, vpacket.size(), srcmask); } srcmask = boost::none; // this is also our return value, even if EDNS0Level == 0 // We only store the localip if needed for fstrm logging ComboAddress localip; - bool dnsOverTLS = false; #ifdef HAVE_FSTRM bool fstrmQEnabled = false; bool fstrmREnabled = false; @@ -342,75 +434,33 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int ret = arecvfrom(buf, 0, ip, &len, qid, domain, type, queryfd, now); } else { - try { - const struct timeval timeout{ g_networkTimeoutMsec / 1000, static_cast(g_networkTimeoutMsec) % 1000 * 1000}; - - Socket s(ip.sin4.sin_family, SOCK_STREAM); - s.setNonBlocking(); - localip = pdns::getQueryLocalAddress(ip.sin4.sin_family, 0); - s.bind(localip); - - std::shared_ptr tlsCtx{nullptr}; - if (SyncRes::s_dot_to_port_853 && ip.getPort() == 853) { - TLSContextParameters tlsParams; - tlsParams.d_provider = "openssl"; - tlsParams.d_validateCertificates = false; - //tlsParams.d_caStore = caaStore; - tlsCtx = getTLSContext(tlsParams); - if (tlsCtx == nullptr) { - g_log << Logger::Error << "DoT to " << ip << " requested but not available" << endl; + bool isNew; + do { + try { + // If we get a new (not re-used) TCP connection that does not + // work, we give up. For reused connections, we assume the + // peer has closed it on error, so we retry. At some point we + // *will* get a new connection, so this loop is not endless. + isNew = true; // tcpconnect() might throw for new connections. In that case, we want to break the loop + isNew = tcpconnect(*now, ip, connection, dnsOverTLS); + ret = tcpsendrecv(ip, connection, localip, vpacket, len, buf); +#ifdef HAVE_FSTRM + if (fstrmQEnabled) { + logFstreamQuery(fstrmLoggers, queryTime, localip, ip, !dnsOverTLS ? DnstapMessage::ProtocolType::DoTCP : DnstapMessage::ProtocolType::DoT, context ? context->d_auth : boost::none, vpacket); } - else { - dnsOverTLS = true; +#endif /* HAVE_FSTRM */ + if (ret == LWResult::Result::Success) { + break; } + connection.d_handler->close(); } - auto handler = std::make_shared("", s.releaseHandle(), timeout, tlsCtx, now->tv_sec); - // Returned state ignored - handler->tryConnect(SyncRes::s_tcp_fast_open_connect, ip); - - uint16_t tlen=htons(vpacket.size()); - char *lenP=(char*)&tlen; - const char *msgP=(const char*)&*vpacket.begin(); - PacketBuffer packet; - packet.reserve(2 + vpacket.size()); - packet.insert(packet.end(), lenP, lenP+2); - packet.insert(packet.end(), msgP, msgP+vpacket.size()); - ret = asendtcp(packet, handler); - if (ret != LWResult::Result::Success) { - handler->close(); - return ret; - } - -#ifdef HAVE_FSTRM - if (fstrmQEnabled) { - logFstreamQuery(fstrmLoggers, queryTime, localip, ip, !dnsOverTLS ? DnstapMessage::ProtocolType::DoTCP : DnstapMessage::ProtocolType::DoT, context ? context->d_auth : boost::none, vpacket); - } -#endif /* HAVE_FSTRM */ - - ret = arecvtcp(packet, 2, handler, false); - if (ret != LWResult::Result::Success) { - return ret; + catch (const NetworkError&) { + ret = LWResult::Result::OSLimitError; // OS limits error } - - memcpy(&tlen, packet.data(), sizeof(tlen)); - len=ntohs(tlen); // switch to the 'len' shared with the rest of the function - - // XXX receive into buf directly? - packet.resize(len); - ret = arecvtcp(packet, len, handler, false); - if (ret != LWResult::Result::Success) { - return ret; + catch (const runtime_error&) { + ret = LWResult::Result::OSLimitError; // OS limits error (PermanentError is transport related) } - - buf.resize(len); - memcpy(buf.data(), packet.data(), len); - - handler->close(); - ret = LWResult::Result::Success; - } - catch (const NetworkError& ne) { - ret = LWResult::Result::OSLimitError; // OS limits error - } + } while (!isNew); } lwr->d_usec=dt.udiff(); @@ -418,7 +468,7 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int if (ret != LWResult::Result::Success) { // includes 'timeout' if (outgoingLoggers) { - logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, srcmask, 0, -1, {}, queryTime, exportTypes); + logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, dnsOverTLS, srcmask, 0, -1, {}, queryTime, exportTypes); } return ret; } @@ -445,7 +495,7 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int if(mdp.d_header.rcode == RCode::FormErr && mdp.d_qname.empty() && mdp.d_qtype == 0 && mdp.d_qclass == 0) { if(outgoingLoggers) { - logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, srcmask, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes); + logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, dnsOverTLS, srcmask, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes); } lwr->d_validpacket = true; return LWResult::Result::Success; // this is "success", the error is set in lwr->d_rcode @@ -489,9 +539,9 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int } if(outgoingLoggers) { - logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, srcmask, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes); + logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, dnsOverTLS, srcmask, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes); } - + lwr->d_validpacket = true; return LWResult::Result::Success; } @@ -505,7 +555,7 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int g_stats.serverParseError++; if(outgoingLoggers) { - logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, srcmask, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes); + logIncomingResponse(outgoingLoggers, context ? context->d_initialRequestId : boost::none, uuid, ip, domain, type, qid, doTCP, dnsOverTLS, srcmask, len, lwr->d_rcode, lwr->d_records, queryTime, exportTypes); } return LWResult::Result::Success; // success - oddly enough @@ -524,3 +574,16 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int return LWResult::Result::PermanentError; } +LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, const std::shared_ptr>>& outgoingLoggers, const std::shared_ptr>>& fstrmLoggers, const std::set& exportTypes, LWResult *lwr, bool* chained) +{ + TCPOutConnectionManager::Connection connection; + auto ret = asyncresolve(ip, domain, type, doTCP, sendRDQuery, EDNS0Level, now, srcmask, context, outgoingLoggers, fstrmLoggers, exportTypes, lwr, chained, connection); + + if (doTCP) { + if (connection.d_handler && lwr->d_validpacket) { + t_tcp_manager.store(*now, ip, std::move(connection)); + } + } + return ret; +} + diff --git a/pdns/malloctrace.cc b/pdns/malloctrace.cc index 961629ba7c4c..4fe49c544cd5 100644 --- a/pdns/malloctrace.cc +++ b/pdns/malloctrace.cc @@ -104,7 +104,7 @@ MallocTracer::allocators_t MallocTracer::topAllocators(int num) l_active=true; allocators_t ret; for(const auto& e : d_stats) { - ret.push_back(make_pair(e.second, e.first)); + ret.emplace_back(e.second, e.first); } std::sort(ret.begin(), ret.end(), [](const allocators_t::value_type& a, diff --git a/pdns/mastercommunicator.cc b/pdns/mastercommunicator.cc index 5f235e50a352..efee904399bf 100644 --- a/pdns/mastercommunicator.cc +++ b/pdns/mastercommunicator.cc @@ -46,27 +46,27 @@ void CommunicatorClass::queueNotifyDomain(const DomainInfo& di, UeberBackend* B) { bool hasQueuedItem=false; - set nsset, ips; + set ips; + set nsset; DNSZoneRecord rr; FindNS fns; - try { if (d_onlyNotify.size()) { B->lookup(QType(QType::NS), di.zone, di.id); while(B->get(rr)) - nsset.insert(getRR(rr.dr)->getNS().toString()); + nsset.insert(getRR(rr.dr)->getNS()); - for(const auto & j : nsset) { - vector nsips=fns.lookup(DNSName(j), B); + for(const auto & ns : nsset) { + vector nsips=fns.lookup(ns, B); if(nsips.empty()) - g_log< l(d_holelock); - d_holes[make_pair(domain,ip)]=time(nullptr); + (*d_holes.lock())[pair(domain,ip)]=time(nullptr); } bool CommunicatorClass::justNotified(const DNSName &domain, const string &ip) { - std::lock_guard l(d_holelock); - if(d_holes.find(make_pair(domain,ip))==d_holes.end()) // no hole + auto holes = d_holes.lock(); + auto it = holes->find(pair(domain,ip)); + if (it == holes->end()) { + // no hole return false; + } - if(d_holes[make_pair(domain,ip)]>time(nullptr)-900) // recent hole + if (it->second > time(nullptr)-900) { + // recent hole return true; + } // do we want to purge this? XXX FIXME return false; diff --git a/pdns/minicurl.cc b/pdns/minicurl.cc index b38413d47a37..91b561cd016b 100644 --- a/pdns/minicurl.cc +++ b/pdns/minicurl.cc @@ -74,7 +74,7 @@ static string extractHostFromURL(const std::string& url) return url.substr(pos, endpos-pos); } -void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen) +void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen, bool verify) { if(rem) { struct curl_slist *hostlist = nullptr; // THIS SHOULD BE FREED @@ -105,8 +105,8 @@ void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const C curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, true); /* only allow HTTP and HTTPS */ curl_easy_setopt(d_curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); - curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, false); - curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, false); + curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, verify); + curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, verify ? 2 : 0); curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, true); curl_easy_setopt(d_curl, CURLOPT_URL, str.c_str()); curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, write_callback); @@ -121,9 +121,9 @@ void MiniCurl::setupURL(const std::string& str, const ComboAddress* rem, const C d_data.clear(); } -std::string MiniCurl::getURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout) +std::string MiniCurl::getURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen, bool verify) { - setupURL(str, rem, src, timeout); + setupURL(str, rem, src, timeout, fastopen, verify); auto res = curl_easy_perform(d_curl); long http_code = 0; curl_easy_getinfo(d_curl, CURLINFO_RESPONSE_CODE, &http_code); @@ -136,9 +136,9 @@ std::string MiniCurl::getURL(const std::string& str, const ComboAddress* rem, co return ret; } -std::string MiniCurl::postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout, bool fastopen) +std::string MiniCurl::postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout, bool fastopen, bool verify) { - setupURL(str, nullptr, nullptr, timeout, fastopen); + setupURL(str, nullptr, nullptr, timeout, fastopen, verify); setHeaders(headers); curl_easy_setopt(d_curl, CURLOPT_POSTFIELDSIZE, postdata.size()); curl_easy_setopt(d_curl, CURLOPT_POSTFIELDS, postdata.c_str()); diff --git a/pdns/minicurl.hh b/pdns/minicurl.hh index c2550d6bd289..b59656ae71d8 100644 --- a/pdns/minicurl.hh +++ b/pdns/minicurl.hh @@ -38,14 +38,14 @@ public: MiniCurl(const string& useragent="MiniCurl/0.0"); ~MiniCurl(); MiniCurl& operator=(const MiniCurl&) = delete; - std::string getURL(const std::string& str, const ComboAddress* rem=nullptr, const ComboAddress* src=nullptr, int timeout = 2); - std::string postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout = 2, bool fastopen = false); + std::string getURL(const std::string& str, const ComboAddress* rem=nullptr, const ComboAddress* src=nullptr, int timeout = 2, bool fastopen = false, bool verify = false); + std::string postURL(const std::string& str, const std::string& postdata, MiniCurlHeaders& headers, int timeout = 2, bool fastopen = false, bool verify = false); private: CURL *d_curl; static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata); std::string d_data; struct curl_slist* d_header_list = nullptr; - void setupURL(const std::string& str, const ComboAddress* rem=nullptr, const ComboAddress* src=nullptr, int timeout = 2, bool fastopen = false); + void setupURL(const std::string& str, const ComboAddress* rem, const ComboAddress* src, int timeout, bool fastopen, bool verify); void setHeaders(const MiniCurlHeaders& headers); void clearHeaders(); }; diff --git a/pdns/misc.cc b/pdns/misc.cc index 0948803af925..c243386a35a7 100644 --- a/pdns/misc.cc +++ b/pdns/misc.cc @@ -67,7 +67,7 @@ size_t writen2(int fd, const void *buf, size_t count) { - const char *ptr = (char*)buf; + const char *ptr = reinterpret_cast(buf); const char *eptr = ptr + count; ssize_t res; @@ -110,7 +110,7 @@ size_t readn2(int fd, void* buffer, size_t len) return len; } -size_t readn2WithTimeout(int fd, void* buffer, size_t len, const struct timeval& idleTimeout, const struct timeval& totalTimeout) +size_t readn2WithTimeout(int fd, void* buffer, size_t len, const struct timeval& idleTimeout, const struct timeval& totalTimeout, bool allowIncomplete) { size_t pos = 0; struct timeval start{0,0}; @@ -123,6 +123,9 @@ size_t readn2WithTimeout(int fd, void* buffer, size_t len, const struct timeval& ssize_t got = read(fd, (char *)buffer + pos, len - pos); if (got > 0) { pos += (size_t) got; + if (allowIncomplete) { + break; + } } else if (got == 0) { throw runtime_error("EOF while reading message"); @@ -165,7 +168,7 @@ size_t writen2WithTimeout(int fd, const void * buffer, size_t len, const struct { size_t pos = 0; do { - ssize_t written = write(fd, (char *)buffer + pos, len - pos); + ssize_t written = write(fd, reinterpret_cast(buffer) + pos, len - pos); if (written > 0) { pos += (size_t) written; @@ -224,7 +227,7 @@ uint32_t getLong(const unsigned char* p) uint32_t getLong(const char* p) { - return getLong((unsigned char *)p); + return getLong(reinterpret_cast(p)); } static bool ciEqual(const string& a, const string& b) @@ -574,6 +577,22 @@ string makeHexDump(const string& str) return ret; } +string makeBytesFromHex(const string &in) { + if (in.size() % 2 != 0) { + throw std::range_error("odd number of bytes in hex string"); + } + string ret; + ret.reserve(in.size()); + unsigned int num; + for (size_t i = 0; i < in.size(); i+=2) { + string numStr = in.substr(i, 2); + num = 0; + sscanf(numStr.c_str(), "%02x", &num); + ret.push_back((uint8_t)num); + } + return ret; +} + void normalizeTV(struct timeval& tv) { if(tv.tv_usec > 1000000) { @@ -726,7 +745,7 @@ int makeIPv4sockaddr(const std::string& str, struct sockaddr_in* ret) if(!*(str.c_str() + pos + 1)) // trailing : return -1; - char *eptr = (char*)str.c_str() + str.size(); + char *eptr = const_cast(str.c_str()) + str.size(); int port = strtol(str.c_str() + pos + 1, &eptr, 10); if (port < 0 || port > 65535) return -1; @@ -1124,26 +1143,79 @@ uint64_t udpErrorStats(const std::string& str) { #ifdef __linux__ ifstream ifs("/proc/net/snmp"); - if(!ifs) + if (!ifs) { return 0; + } + string line; - vector parts; - while(getline(ifs,line)) { - if(boost::starts_with(line, "Udp: ") && isdigit(line[5])) { + while (getline(ifs, line)) { + if (boost::starts_with(line, "Udp: ") && isdigit(line.at(5))) { + vector parts; stringtok(parts, line, " \n\t\r"); - if(parts.size() < 7) - break; - if(str=="udp-rcvbuf-errors") - return std::stoull(parts[5]); - else if(str=="udp-sndbuf-errors") - return std::stoull(parts[6]); - else if(str=="udp-noport-errors") - return std::stoull(parts[2]); - else if(str=="udp-in-errors") - return std::stoull(parts[3]); - else - return 0; + + if (parts.size() < 7) { + break; + } + + if (str == "udp-rcvbuf-errors") { + return std::stoull(parts.at(5)); + } + else if (str == "udp-sndbuf-errors") { + return std::stoull(parts.at(6)); + } + else if (str == "udp-noport-errors") { + return std::stoull(parts.at(2)); + } + else if (str == "udp-in-errors") { + return std::stoull(parts.at(3)); + } + else if (parts.size() >= 8 && str == "udp-in-csum-errors") { + return std::stoull(parts.at(7)); + } + else { + return 0; + } + } + } +#endif + return 0; +} + +uint64_t udp6ErrorStats(const std::string& str) +{ +#ifdef __linux__ + const std::map keys = { + { "udp6-in-errors", "Udp6InErrors" }, + { "udp6-recvbuf-errors", "Udp6RcvbufErrors" }, + { "udp6-sndbuf-errors", "Udp6SndbufErrors" }, + { "udp6-noport-errors", "Udp6NoPorts" }, + { "udp6-in-csum-errors", "Udp6InCsumErrors" } + }; + + auto key = keys.find(str); + if (key == keys.end()) { + return 0; + } + + ifstream ifs("/proc/net/snmp6"); + if (!ifs) { + return 0; + } + + std::string line; + while (getline(ifs, line)) { + if (!boost::starts_with(line, key->second)) { + continue; } + + std::vector parts; + stringtok(parts, line, " \n\t\r"); + + if (parts.size() != 2) { + return 0; + } + + return std::stoull(parts.at(1)); } #endif return 0; @@ -1282,6 +1354,9 @@ uint64_t getOpenFileDescriptors(const std::string&) closedir(dirhdl); return ret; +#elif defined(__OpenBSD__) + // FreeBSD also has this in libopenbsd, but I don't know if that's available always + return getdtablecount(); #else return 0; #endif @@ -1642,3 +1717,36 @@ size_t parseSVCBValueList(const std::string &in, vector &val) { parseSVCBValueListFromParsedRFC1035CharString(parsed, val); return ret; }; + +#ifdef HAVE_CRYPTO_MEMCMP +#include +#else /* HAVE_CRYPTO_MEMCMP */ +#ifdef HAVE_SODIUM_MEMCMP +#include +#endif /* HAVE_SODIUM_MEMCMP */ +#endif /* HAVE_CRYPTO_MEMCMP */ + +bool constantTimeStringEquals(const std::string& a, const std::string& b) +{ + if (a.size() != b.size()) { + return false; + } + const size_t size = a.size(); +#ifdef HAVE_CRYPTO_MEMCMP + return CRYPTO_memcmp(a.c_str(), b.c_str(), size) == 0; +#else /* HAVE_CRYPTO_MEMCMP */ +#ifdef HAVE_SODIUM_MEMCMP + return sodium_memcmp(a.c_str(), b.c_str(), size) == 0; +#else /* HAVE_SODIUM_MEMCMP */ + const volatile unsigned char *_a = (const volatile unsigned char *) a.c_str(); + const volatile unsigned char *_b = (const volatile unsigned char *) b.c_str(); + unsigned char res = 0; + + for (size_t idx = 0; idx < size; idx++) { + res |= _a[idx] ^ _b[idx]; + } + + return res == 0; +#endif /* !HAVE_SODIUM_MEMCMP */ +#endif /* !HAVE_CRYPTO_MEMCMP */ +} diff --git a/pdns/misc.hh b/pdns/misc.hh index 0372e8ffae41..f08be7591c0d 100644 --- a/pdns/misc.hh +++ b/pdns/misc.hh @@ -136,10 +136,10 @@ vstringtok (Container &container, string const &in, // push token if (j == string::npos) { - container.push_back (make_pair(i, len)); + container.emplace_back(i, len); return; } else - container.push_back (make_pair(i, j)); + container.emplace_back(i, j); // set up for next loop i = j + 1; @@ -149,7 +149,7 @@ vstringtok (Container &container, string const &in, size_t writen2(int fd, const void *buf, size_t count); inline size_t writen2(int fd, const std::string &s) { return writen2(fd, s.data(), s.size()); } size_t readn2(int fd, void* buffer, size_t len); -size_t readn2WithTimeout(int fd, void* buffer, size_t len, const struct timeval& idleTimeout, const struct timeval& totalTimeout={0,0}); +size_t readn2WithTimeout(int fd, void* buffer, size_t len, const struct timeval& idleTimeout, const struct timeval& totalTimeout={0,0}, bool allowIncomplete=false); size_t writen2WithTimeout(int fd, const void * buffer, size_t len, const struct timeval& timeout); void toLowerInPlace(string& str); @@ -304,6 +304,8 @@ inline void unixDie(const string &why) } string makeHexDump(const string& str); +//! Convert the hexstring in to a byte string +string makeBytesFromHex(const string &in); void normalizeTV(struct timeval& tv); const struct timeval operator+(const struct timeval& lhs, const struct timeval& rhs); @@ -565,6 +567,7 @@ size_t getPipeBufferSize(int fd); bool setPipeBufferSize(int fd, size_t size); uint64_t udpErrorStats(const std::string& str); +uint64_t udp6ErrorStats(const std::string& str); uint64_t tcpErrorStats(const std::string& str); uint64_t getRealMemoryUsage(const std::string&); uint64_t getSpecialMemoryUsage(const std::string&); @@ -589,18 +592,21 @@ T valueOrEmpty(const P val) { // I'm not very OCD, but I appreciate loglines like "processing 1 delta", "processing 2 deltas" :-) -template -const char* addS(Integer siz, typename std::enable_if::value>::type*P=0) +template ::value, bool> = true> +const char* addS(Integer siz, const char* singular = "", const char *plural = "s") { - if(!siz || siz > 1) - return "s"; - else return ""; + if (siz == 1) { + return singular; + } + return plural; } -template -const char* addS(const C& c, typename std::enable_if::value>::type*P=0) +template ::value, bool> = true> +const char* addS(const C& c, const char* singular = "", const char *plural = "s") { - return addS(c.size()); + return addS(c.size(), singular, plural); } template @@ -633,5 +639,54 @@ size_t parseSVCBValueList(const std::string &in, vector &val); std::string makeLuaString(const std::string& in); +bool constantTimeStringEquals(const std::string& a, const std::string& b); + // Used in NID and L64 records struct NodeOrLocatorID { uint8_t content[8]; }; + +struct FDWrapper +{ + FDWrapper() + { + } + + FDWrapper(int desc): d_fd(desc) + { + } + + ~FDWrapper() + { + if (d_fd != -1) { + close(d_fd); + d_fd = -1; + } + } + + FDWrapper(FDWrapper&& rhs): d_fd(rhs.d_fd) + { + rhs.d_fd = -1; + } + + FDWrapper& operator=(FDWrapper&& rhs) + { + if (d_fd != -1) { + close(d_fd); + } + d_fd = rhs.d_fd; + rhs.d_fd = -1; + return *this; + } + + int getHandle() const + { + return d_fd; + } + + operator int() const + { + return d_fd; + } + +private: + int d_fd{-1}; +}; diff --git a/pdns/mplexer.hh b/pdns/mplexer.hh index b5742a8c48cb..d85efe91fe54 100644 --- a/pdns/mplexer.hh +++ b/pdns/mplexer.hh @@ -204,7 +204,7 @@ public: if (it->d_ttd.tv_sec == 0 || tied <= boost::tie(it->d_ttd.tv_sec, it->d_ttd.tv_usec)) { break; } - ret.push_back(std::make_pair(it->d_fd, it->d_parameter)); + ret.emplace_back(it->d_fd, it->d_parameter); } return ret; diff --git a/pdns/nameserver.cc b/pdns/nameserver.cc index c52341bce41e..ce07abdc7d8f 100644 --- a/pdns/nameserver.cc +++ b/pdns/nameserver.cc @@ -32,6 +32,7 @@ #include #include "responsestats.hh" +#include "common_startup.hh" #include "dns.hh" #include "dnsbackend.hh" #include "dnspacket.hh" @@ -40,6 +41,7 @@ #include "logger.hh" #include "arguments.hh" #include "statbag.hh" +#include "proxy-protocol.hh" #include "namespaces.hh" @@ -116,11 +118,11 @@ void UDPNameserver::bindAddresses() throw PDNSException("Unable to set UDP socket " + locala.toStringWithPort() + " to non-blocking: "+stringerror()); if(IsAnyAddress(locala)) { - setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one)); + (void)setsockopt(s, IPPROTO_IP, GEN_IP_PKTINFO, &one, sizeof(one)); if (locala.isIPv6()) { - setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &one, sizeof(one)); // if this fails, we report an error in tcpreceiver too + (void)setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &one, sizeof(one)); // if this fails, we report an error in tcpreceiver too #ifdef IPV6_RECVPKTINFO - setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one)); + (void)setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, &one, sizeof(one)); #endif } } @@ -301,11 +303,31 @@ bool UDPNameserver::receive(DNSPacket& packet, std::string& buffer) packet.d_dt.setTimeval(recvtv); } else - packet.d_dt.set(); // timing + packet.d_dt.set(); // timing + + if (g_proxyProtocolACL.match(remote)) { + ComboAddress psource, pdestination; + bool proxyProto, tcp; + std::vector ppvalues; + + buffer.resize(len); + ssize_t used = parseProxyHeader(buffer, proxyProto, psource, pdestination, tcp, ppvalues); + if (used <= 0 || (size_t) used > g_proxyProtocolMaximumSize || (len - used) > DNSPacket::s_udpTruncationThreshold) { + S.inc("corrupt-packets"); + S.ringAccount("remotes-corrupt", packet.d_remote); + return false; + } + buffer.erase(0, used); + packet.d_inner_remote = psource; + packet.d_tcp = tcp; + } + else { + packet.d_inner_remote.reset(); + } if(packet.parse(&buffer.at(0), (size_t) len)<0) { S.inc("corrupt-packets"); - S.ringAccount("remotes-corrupt", packet.d_remote); + S.ringAccount("remotes-corrupt", packet.getInnerRemote()); return false; // unable to parse } diff --git a/pdns/namespaces.hh b/pdns/namespaces.hh index 5e7a7d6fa3a2..9e1f3adaa8da 100644 --- a/pdns/namespaces.hh +++ b/pdns/namespaces.hh @@ -42,7 +42,6 @@ using std::clog; using std::cout; using std::endl; using std::ifstream; -using std::make_pair; using std::make_unique; using std::map; using std::max; diff --git a/pdns/nsec3dig.cc b/pdns/nsec3dig.cc index 44a8d0a39069..13dcdf057194 100644 --- a/pdns/nsec3dig.cc +++ b/pdns/nsec3dig.cc @@ -142,7 +142,7 @@ try throw PDNSException("tcp read failed"); len=ntohs(len); - std::unique_ptr creply(new char[len]); + auto creply = std::make_unique(len); int n=0; int numread; while(nfirst.d_name.getRawLabel(0)), toBase32Hex(r->d_nexthash))); + nsec3s.emplace(toLower(i->first.d_name.getRawLabel(0)), toBase32Hex(r->d_nexthash)); nsec3salt = r->d_salt; nsec3iters = r->d_iterations; - nsec3t.insert(make_pair(toLower(i->first.d_name.getRawLabel(0)), r->numberOfTypesSet())); + nsec3t.emplace(toLower(i->first.d_name.getRawLabel(0)), r->numberOfTypesSet()); } else { diff --git a/pdns/opensslsigners.cc b/pdns/opensslsigners.cc index 0f5396cea225..7138f9705285 100644 --- a/pdns/opensslsigners.cc +++ b/pdns/opensslsigners.cc @@ -262,14 +262,14 @@ DNSCryptoKeyEngine::storvector_t OpenSSLRSADNSCryptoKeyEngine::convertToISCVecto RSA_get0_key(d_key.get(), &n, &e, &d); RSA_get0_factors(d_key.get(), &p, &q); RSA_get0_crt_params(d_key.get(), &dmp1, &dmq1, &iqmp); - outputs.push_back(make_pair("Modulus", n)); - outputs.push_back(make_pair("PublicExponent", e)); - outputs.push_back(make_pair("PrivateExponent", d)); - outputs.push_back(make_pair("Prime1", p)); - outputs.push_back(make_pair("Prime2", q)); - outputs.push_back(make_pair("Exponent1", dmp1)); - outputs.push_back(make_pair("Exponent2", dmq1)); - outputs.push_back(make_pair("Coefficient", iqmp)); + outputs.emplace_back("Modulus", n); + outputs.emplace_back("PublicExponent", e); + outputs.emplace_back("PrivateExponent", d); + outputs.emplace_back("Prime1", p); + outputs.emplace_back("Prime2", q); + outputs.emplace_back("Exponent1", dmp1); + outputs.emplace_back("Exponent2", dmq1); + outputs.emplace_back("Coefficient", iqmp); string algorithm=std::to_string(d_algorithm); switch(d_algorithm) { @@ -286,7 +286,7 @@ DNSCryptoKeyEngine::storvector_t OpenSSLRSADNSCryptoKeyEngine::convertToISCVecto default: algorithm += " (?)"; } - storvect.push_back(make_pair("Algorithm", algorithm)); + storvect.emplace_back("Algorithm", algorithm); for(const outputs_t::value_type& value : outputs) { std::string tmp; @@ -294,7 +294,7 @@ DNSCryptoKeyEngine::storvector_t OpenSSLRSADNSCryptoKeyEngine::convertToISCVecto int len = BN_bn2bin(value.second, reinterpret_cast(&tmp.at(0))); if (len >= 0) { tmp.resize(len); - storvect.push_back(make_pair(value.first, tmp)); + storvect.emplace_back(value.first, tmp); } } @@ -640,7 +640,7 @@ DNSCryptoKeyEngine::storvector_t OpenSSLECDSADNSCryptoKeyEngine::convertToISCVec else algorithm = " ? (?)"; - storvect.push_back(make_pair("Algorithm", algorithm)); + storvect.emplace_back("Algorithm", algorithm); const BIGNUM *key = EC_KEY_get0_private_key(d_eckey.get()); if (key == nullptr) { @@ -655,7 +655,7 @@ DNSCryptoKeyEngine::storvector_t OpenSSLECDSADNSCryptoKeyEngine::convertToISCVec if (d_len - len) prefix.append(d_len - len, 0x00); - storvect.push_back(make_pair("PrivateKey", prefix + tmp)); + storvect.emplace_back("PrivateKey", prefix + tmp); return storvect; } @@ -943,7 +943,7 @@ DNSCryptoKeyEngine::storvector_t OpenSSLEDDSADNSCryptoKeyEngine::convertToISCVec algorithm = " ? (?)"; } - storvect.push_back(make_pair("Algorithm", algorithm)); + storvect.emplace_back("Algorithm", algorithm); string buf; size_t len = d_len; @@ -951,7 +951,7 @@ DNSCryptoKeyEngine::storvector_t OpenSSLEDDSADNSCryptoKeyEngine::convertToISCVec if (EVP_PKEY_get_raw_private_key(d_edkey.get(), reinterpret_cast(&buf.at(0)), &len) < 1) { throw runtime_error(getName() + " Could not get private key from d_edkey"); } - storvect.push_back(make_pair("PrivateKey", buf)); + storvect.emplace_back("PrivateKey", buf); return storvect; } diff --git a/pdns/packetcache.hh b/pdns/packetcache.hh index 8aa6933f4d32..15f9d2de4642 100644 --- a/pdns/packetcache.hh +++ b/pdns/packetcache.hh @@ -31,9 +31,9 @@ public: /* hash the packet from the provided position, which should point right after tje qname. This skips: - the query ID ; - EDNS Cookie options, if any ; - - EDNS Client Subnet options, if any and skipECS is true. + - Any given option code present in optionsToSkip */ - static uint32_t hashAfterQname(const pdns_string_view& packet, uint32_t currentHash, size_t pos, bool skipECS) + static uint32_t hashAfterQname(const pdns_string_view& packet, uint32_t currentHash, size_t pos, const std::unordered_set& optionsToSkip = {EDNSOptionCode::COOKIE}) { const size_t packetSize = packet.size(); assert(packetSize >= sizeof(dnsheader)); @@ -82,22 +82,12 @@ public: return currentHash; } - bool skip = false; - if (optionCode == EDNSOptionCode::COOKIE) { - skip = true; - } - else if (optionCode == EDNSOptionCode::ECS) { - if (skipECS) { - skip = true; - } - } - - if (!skip) { + if (optionsToSkip.count(optionCode) == 0) { /* hash the option code, length and content */ currentHash = burtle(reinterpret_cast(&packet.at(pos)), 4 + optionLen, currentHash); } else { - /* hash the option code and length */ + /* skip option: hash only its code and length */ currentHash = burtle(reinterpret_cast(&packet.at(pos)), 4, currentHash); } @@ -140,9 +130,9 @@ public: /* hash the packet from the beginning, including the qname. This skips: - the query ID ; - EDNS Cookie options, if any ; - - EDNS Client Subnet options, if any and skipECS is true. + - Any given option code present in optionsToSkip */ - static uint32_t canHashPacket(const std::string& packet, bool skipECS) + static uint32_t canHashPacket(const std::string& packet, const std::unordered_set& optionsToSkip = {EDNSOptionCode::COOKIE}) { size_t pos = 0; uint32_t currentHash = hashHeaderAndQName(packet, pos); @@ -152,7 +142,7 @@ public: return currentHash; } - return hashAfterQname(packet, currentHash, pos, skipECS); + return hashAfterQname(packet, currentHash, pos, optionsToSkip); } static bool queryHeaderMatches(const std::string& cachedQuery, const std::string& query) diff --git a/pdns/packethandler.cc b/pdns/packethandler.cc index 45164c35d813..d59f7881dd65 100644 --- a/pdns/packethandler.cc +++ b/pdns/packethandler.cc @@ -76,7 +76,7 @@ PacketHandler::PacketHandler():B(s_programname), d_dk(&B) } else { - d_pdl = std::unique_ptr(new AuthLua4()); + d_pdl = std::make_unique(); d_pdl->loadFile(fname); } fname = ::arg()["lua-dnsupdate-policy-script"]; @@ -86,7 +86,7 @@ PacketHandler::PacketHandler():B(s_programname), d_dk(&B) } else { - d_update_policy_lua = std::unique_ptr(new AuthLua4()); + d_update_policy_lua = std::make_unique(); d_update_policy_lua->loadFile(fname); } } @@ -340,9 +340,9 @@ vector PacketHandler::getBestReferralNS(DNSPacket& p, const DNSNa return ret; } -vector PacketHandler::getBestDNAMESynth(DNSPacket& p, DNSName &target) +void PacketHandler::getBestDNAMESynth(DNSPacket& p, DNSName &target, vector &ret) { - vector ret; + ret.clear(); DNSZoneRecord rr; DNSName prefix; DNSName subdomain(target); @@ -356,18 +356,18 @@ vector PacketHandler::getBestDNAMESynth(DNSPacket& p, DNSName &ta rr.dr.d_name = prefix + rr.dr.d_name; rr.dr.d_content = std::make_shared(CNAMERecordContent(prefix + getRR(rr.dr)->getTarget())); rr.auth = false; // don't sign CNAME - target= getRR(rr.dr)->getTarget(); + target = getRR(rr.dr)->getTarget(); ret.push_back(rr); } if(!ret.empty()) - return ret; + return; if(subdomain.countLabels()) prefix.appendRawLabel(subdomain.getRawLabels()[0]); // XXX DNSName pain this feels wrong if(subdomain == d_sd.qname) // stop at SOA break; } while( subdomain.chopOff() ); // 'www.powerdns.org' -> 'powerdns.org' -> 'org' -> '' - return ret; + return; } @@ -941,10 +941,13 @@ int PacketHandler::trySuperMaster(const DNSPacket& p, const DNSName& tsigkeyname int PacketHandler::trySuperMasterSynchronous(const DNSPacket& p, const DNSName& tsigkeyname) { - ComboAddress remote = p.getRemote(); + ComboAddress remote = p.getInnerRemote(); if(p.hasEDNSSubnet() && pdns::isAddressTrustedNotificationProxy(remote)) { remote = p.getRealRemote().getNetwork(); } + else { + remote = p.getInnerRemote(); + } remote.setPort(53); Resolver::res_t nsset; @@ -1019,34 +1022,34 @@ int PacketHandler::processNotify(const DNSPacket& p) if master is higher -> do stuff */ - g_log< meta; if (B.getDomainMetadata(p.qdomain,"AXFR-MASTER-TSIG",meta) && meta.size() > 0) { DNSName expected{meta[0]}; if (p.getTSIGKeyname() != expected) { - g_log< forwardNotify(s_forwardNotify); for(const auto & j : forwardNotify) { - g_log<& r, DNSNam if(!d_doDNAME) return false; DLOG(g_log< rrset = getBestDNAMESynth(p, target); - if(!rrset.empty()) { - for(auto& rr: rrset) { - rr.dr.d_place = DNSResourceRecord::ANSWER; - r->addRecord(std::move(rr)); + vector rrset; + try { + getBestDNAMESynth(p, target, rrset); + if(!rrset.empty()) { + for(size_t i = 0; i < rrset.size(); i++) { + rrset.at(i).dr.d_place = DNSResourceRecord::ANSWER; + r->addRecord(std::move(rrset.at(i))); + } + return true; + } + } catch (const std::range_error &e) { + // Add the DNAME regardless, but throw to let the caller know we could not + // synthesize a CNAME + if(!rrset.empty()) { + for(size_t i = 0; i < rrset.size(); i++) { + rrset.at(i).dr.d_place = DNSResourceRecord::ANSWER; + r->addRecord(std::move(rrset.at(i))); + } } - return true; + throw e; } return false; } @@ -1273,26 +1289,40 @@ std::unique_ptr PacketHandler::doQuestion(DNSPacket& p) if(p.d.qr) { // QR bit from dns packet (thanks RA from N) if(d_logDNSDetails) - g_log< 0) { - r = p.replyPacket(); + if (p.hasEDNS()) { + if(p.getEDNSVersion() > 0) { + r = p.replyPacket(); - // PacketWriter::addOpt will take care of setting this correctly in the packet - r->setEDNSRcode(ERCode::BADVERS); - return r; + // PacketWriter::addOpt will take care of setting this correctly in the packet + r->setEDNSRcode(ERCode::BADVERS); + return r; + } + if (p.hasEDNSCookie()) { + if (!p.hasWellFormedEDNSCookie()) { + r = p.replyPacket(); + r->setRcode(RCode::FormErr); + return r; + } + if (!p.hasValidEDNSCookie() && !p.d_tcp) { + r = p.replyPacket(); + r->setEDNSRcode(ERCode::BADCOOKIE); + return r; + } + } } if(p.d_havetsig) { @@ -1329,9 +1359,9 @@ std::unique_ptr PacketHandler::doQuestion(DNSPacket& p) if(!validDNSName(p.qdomain)) { if(d_logDNSDetails) - g_log<setRcode(RCode::ServFail); return r; @@ -1359,13 +1389,13 @@ std::unique_ptr PacketHandler::doQuestion(DNSPacket& p) return nullptr; } - g_log<setRcode(RCode::NotImp); return r; } - // g_log<setRcode(RCode::Refused); @@ -1464,7 +1494,7 @@ std::unique_ptr PacketHandler::doQuestion(DNSPacket& p) // this TRUMPS a cname! if(p.qtype.getCode() == QType::RRSIG) { - g_log<setRcode(RCode::Refused); goto sendit; } @@ -1630,16 +1660,20 @@ std::unique_ptr PacketHandler::doQuestion(DNSPacket& p) goto sendit; } - else if(tryDNAME(p, r, target)) { - retargetcount++; - goto retargeted; - } - else - { - if (!(((p.qtype.getCode() == QType::CNAME) || (p.qtype.getCode() == QType::ANY)) && retargetcount > 0)) - makeNXDomain(p, r, target, wildcard); + try { + if (tryDNAME(p, r, target)) { + retargetcount++; + goto retargeted; + } + } catch (const std::range_error &e) { + // We couldn't make a CNAME..... + r->setRcode(RCode::YXDomain); + goto sendit; } - + + if (!(((p.qtype.getCode() == QType::CNAME) || (p.qtype.getCode() == QType::ANY)) && retargetcount > 0)) + makeNXDomain(p, r, target, wildcard); + goto sendit; } diff --git a/pdns/packethandler.hh b/pdns/packethandler.hh index 01b4c403e9a4..e9e79f391fbd 100644 --- a/pdns/packethandler.hh +++ b/pdns/packethandler.hh @@ -97,7 +97,7 @@ private: void makeNXDomain(DNSPacket& p, std::unique_ptr& r, const DNSName& target, const DNSName& wildcard); void makeNOError(DNSPacket& p, std::unique_ptr& r, const DNSName& target, const DNSName& wildcard, int mode); vector getBestReferralNS(DNSPacket& p, const DNSName &target); - vector getBestDNAMESynth(DNSPacket& p, DNSName &target); + void getBestDNAMESynth(DNSPacket& p, DNSName &target, vector &ret); bool tryDNAME(DNSPacket& p, std::unique_ptr& r, DNSName &target); bool tryReferral(DNSPacket& p, std::unique_ptr& r, const DNSName &target, bool retargeted); diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 3efc32837575..ea455bb48248 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -98,8 +98,10 @@ #include "shuffle.hh" #ifdef NOD_ENABLED #include "nod.hh" +#include "logging.hh" #endif /* NOD_ENABLED */ #include "query-local-address.hh" +#include "rec-tcpout.hh" #include "rec-snmp.hh" #include "rec-taskqueue.hh" @@ -114,6 +116,7 @@ #include "rec-protozero.hh" #include "xpf.hh" +#include "rec-eventtrace.hh" typedef map tcpClientCounts_t; @@ -140,6 +143,8 @@ thread_local FDMultiplexer* t_fdm{nullptr}; thread_local std::unique_ptr t_remotes, t_servfailremotes, t_largeanswerremotes, t_bogusremotes; thread_local std::unique_ptr > > t_queryring, t_servfailqueryring, t_bogusqueryring; thread_local std::shared_ptr t_allowFrom; +thread_local std::shared_ptr t_allowNotifyFrom; +thread_local std::shared_ptr t_allowNotifyFor; #ifdef NOD_ENABLED thread_local std::shared_ptr t_nodDBp; thread_local std::shared_ptr t_udrDBp; @@ -198,9 +203,11 @@ enum class PaddingMode { Always, PaddedQueries }; static listenSocketsAddresses_t g_listenSocketsAddresses; // is shared across all threads right now static set g_fromtosockets; // listen sockets that use 'sendfromto()' mechanism (without actually using sendfromto()) -static AtomicCounter counter; +static std::atomic counter; static std::shared_ptr g_initialDomainMap; // new threads needs this to be setup static std::shared_ptr g_initialAllowFrom; // new thread needs to be setup with this +static std::shared_ptr g_initialAllowNotifyFrom; // new threads need this to be setup +static std::shared_ptr g_initialAllowNotifyFor; // new threads need this to be setup static NetmaskGroup g_XPFAcl; static NetmaskGroup g_proxyProtocolACL; static NetmaskGroup g_paddingFrom; @@ -278,13 +285,19 @@ GlobalStateHolder g_DoTToAuthNames; #define BAD_NETS "0.0.0.0/8, 192.0.0.0/24, 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24, 240.0.0.0/4, ::/96, ::ffff:0:0/96, 100::/64, 2001:db8::/32" #define DONT_QUERY LOCAL_NETS ", " BAD_NETS +struct ThreadMSG +{ + pipefunc_t func; + bool wantAnswer; +}; + //! used to send information to a newborn mthread struct DNSComboWriter { - DNSComboWriter(const std::string& query, const struct timeval& now): d_mdp(true, query), d_now(now), d_query(query) + DNSComboWriter(const std::string& query, const struct timeval& now, shared_ptr luaContext): d_mdp(true, query), d_now(now), d_query(query), d_luaContext(luaContext) { } - DNSComboWriter(const std::string& query, const struct timeval& now, std::unordered_set&& policyTags, LuaContext::LuaObject&& data, std::vector&& records): d_mdp(true, query), d_now(now), d_query(query), d_policyTags(std::move(policyTags)), d_records(std::move(records)), d_data(std::move(data)) + DNSComboWriter(const std::string& query, const struct timeval& now, std::unordered_set&& policyTags, shared_ptr luaContext, LuaContext::LuaObject&& data, std::vector&& records): d_mdp(true, query), d_now(now), d_query(query), d_policyTags(std::move(policyTags)), d_records(std::move(records)), d_luaContext(luaContext), d_data(std::move(data)) { } @@ -336,6 +349,7 @@ struct DNSComboWriter { d_local holds our own. */ ComboAddress d_local; ComboAddress d_destination; + RecEventTrace d_eventTrace; boost::uuids::uuid d_uuid; string d_requestorId; string d_deviceId; @@ -345,7 +359,11 @@ struct DNSComboWriter { std::unordered_set d_policyTags; std::string d_routingTag; std::vector d_records; + + // d_data is tied to this LuaContext so we need to keep it alive and use it, not a newer one, as long as d_data exists + shared_ptr d_luaContext; LuaContext::LuaObject d_data; + EDNSSubnetOpts d_ednssubnet; shared_ptr d_tcpConnection; boost::optional d_extendedErrorCode{boost::none}; @@ -853,7 +871,7 @@ TCPConnection::~TCPConnection() --s_currentConnections; } -AtomicCounter TCPConnection::s_currentConnections; +std::atomic TCPConnection::s_currentConnections; static void terminateTCPConnection(int fd) { @@ -991,7 +1009,7 @@ static void updateResponseStats(int res, const ComboAddress& remote, unsigned in if(t_servfailremotes) { t_servfailremotes->push_back(remote); if(query && t_servfailqueryring) // packet cache - t_servfailqueryring->push_back(make_pair(*query, qtype)); + t_servfailqueryring->push_back({*query, qtype}); } g_stats.servFails++; break; @@ -1026,7 +1044,7 @@ static void protobufLogQuery(LocalStateHolder& luaconfsLocal, co pdns::ProtoZero::RecMessage m{128, std::string::size_type(policyTags.empty() ? 0 : 64)}; // It's a guess m.setType(pdns::ProtoZero::Message::MessageType::DNSQueryType); - m.setRequest(uniqueId, requestor, local, qname, qtype, qclass, id, tcp, len); + m.setRequest(uniqueId, requestor, local, qname, qtype, qclass, id, tcp ? pdns::ProtoZero::Message::TransportProtocol::TCP : pdns::ProtoZero::Message::TransportProtocol::UDP, len); m.setServerIdentity(SyncRes::s_serverID); m.setEDNSSubnet(ednssubnet, ednssubnet.isIPv4() ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); m.setRequestorId(requestorId); @@ -1063,7 +1081,8 @@ static void protobufLogResponse(const struct dnsheader* dh, LocalStateHolder& meta) + const string& deviceName, const std::map& meta, + const RecEventTrace& eventTrace) { pdns::ProtoZero::RecMessage pbMessage(pbData ? pbData->d_message : "", pbData ? pbData->d_response : "", 64, 10); // The extra bytes we are going to add // Normally we take the immutable string from the cache and append a few values, but if it's not there (can this happen?) @@ -1087,7 +1106,7 @@ static void protobufLogResponse(const struct dnsheader* dh, LocalStateHolderid); pbMessage.setTime(); @@ -1105,6 +1124,9 @@ static void protobufLogResponse(const struct dnsheader* dh, LocalStateHolder& luaconfsLoc #endif /* HAVE_FSTRM */ #ifdef NOD_ENABLED -static bool nodCheckNewDomain(const DNSName& dname) +static bool nodCheckNewDomain(const shared_ptr& nodlogger, const DNSName& dname) { bool ret = false; // First check the (sub)domain isn't ignored for NOD purposes @@ -1396,7 +1418,8 @@ static bool nodCheckNewDomain(const DNSName& dname) if (t_nodDBp && t_nodDBp->isNewDomain(dname)) { if (g_nodLog) { // This should probably log to a dedicated log file - g_log<& nodlogger, const DNSName& dname) { if (!(g_nodLookupDomain.isRoot())) { // Send a DNS A query to .g_nodLookupDomain @@ -1413,15 +1436,17 @@ static void sendNODLookup(const DNSName& dname) qname = dname + g_nodLookupDomain; } catch(const std::range_error &e) { + nodlogger->v(10)->error(Logr::Error, "DNSName too long", "Unable to send NOD lookup"); ++g_stats.nodLookupsDroppedOversize; return; } + nodlogger->v(10)->info(Logr::Debug, "Sending NOD lookup", "nodqname", Logging::Loggable(qname)); vector dummy; directResolve(qname, QType::A, QClass::IN, dummy, nullptr, false); } } -static bool udrCheckUniqueDNSRecord(const DNSName& dname, uint16_t qtype, const DNSRecord& record) +static bool udrCheckUniqueDNSRecord(const shared_ptr& nodlogger, const DNSName& dname, uint16_t qtype, const DNSRecord& record) { bool ret = false; if (record.d_place == DNSResourceRecord::ANSWER || @@ -1431,8 +1456,15 @@ static bool udrCheckUniqueDNSRecord(const DNSName& dname, uint16_t qtype, const ss << dname.toDNSStringLC() << ":" << qtype << ":" << qtype << ":" << record.d_type << ":" << record.d_name.toDNSStringLC() << ":" << record.d_content->getZoneRepresentation(); if (t_udrDBp && t_udrDBp->isUniqueResponse(ss.str())) { if (g_udrLog) { - // This should also probably log to a dedicated file. - g_log<getZoneRepresentation()) + ); + ); } ret = true; } @@ -1441,6 +1473,8 @@ static bool udrCheckUniqueDNSRecord(const DNSName& dname, uint16_t qtype, const } #endif /* NOD_ENABLED */ +static bool answerIsNOData(uint16_t requestedType, int rcode, const std::vector& records); + int followCNAMERecords(vector& ret, const QType qtype, int rcode) { vector resolved; @@ -1461,8 +1495,14 @@ int followCNAMERecords(vector& ret, const QType qtype, int rcode) rcode = directResolve(target, qtype, QClass::IN, resolved, t_pdl); - for(DNSRecord& rr : resolved) { - ret.push_back(std::move(rr)); + if (g_dns64Prefix && qtype == QType::AAAA && answerIsNOData(qtype, rcode, resolved)) { + rcode = getFakeAAAARecords(target, *g_dns64Prefix, resolved); + } + + for (DNSRecord& rr : resolved) { + if (rr.d_place == DNSResourceRecord::ANSWER) { + ret.push_back(std::move(rr)); + } } return rcode; } @@ -1578,12 +1618,27 @@ static bool answerIsNOData(uint16_t requestedType, int rcode, const std::vector< return true; } +static bool isAllowNotifyForZone(DNSName qname) +{ + if (t_allowNotifyFor->empty()) { + return false; + } + + notifyset_t::const_iterator ret; + do { + ret = t_allowNotifyFor->find(qname); + if (ret != t_allowNotifyFor->end()) + return true; + } while (qname.chopOff()); + return false; +} + static void startDoResolve(void *p) { auto dc=std::unique_ptr(reinterpret_cast(p)); try { if (t_queryring) - t_queryring->push_back(make_pair(dc->d_mdp.d_qname, dc->d_mdp.d_qtype)); + t_queryring->push_back({dc->d_mdp.d_qname, dc->d_mdp.d_qtype}); uint16_t maxanswersize = dc->d_tcp ? 65535 : min(static_cast(512), g_udpTruncationThreshold); EDNSOpts edo; @@ -1594,6 +1649,10 @@ static void startDoResolve(void *p) bool addPaddingToResponse = false; #ifdef NOD_ENABLED bool hasUDR = false; + std::shared_ptr nodlogger{nullptr}; + if (g_udrEnabled || g_nodEnabled) { + nodlogger = g_slog->withName("nod")->v(1)->withValues("qname", Logging::Loggable(dc->d_mdp.d_qname)); + } #endif /* NOD_ENABLED */ DNSPacketWriter::optvect_t returnedEdnsOptions; // Here we stuff all the options for the return packet uint8_t ednsExtRCode = 0; @@ -1626,7 +1685,7 @@ static void startDoResolve(void *p) const static string mode_server_id = ::arg()["server-id"]; if (mode_server_id != "disabled" && !mode_server_id.empty() && maxanswersize > (EDNSOptionCodeSize + EDNSOptionLengthSize + mode_server_id.size())) { - returnedEdnsOptions.push_back(make_pair(EDNSOptionCode::NSID, mode_server_id)); + returnedEdnsOptions.emplace_back(EDNSOptionCode::NSID, mode_server_id); variableAnswer = true; // Can't packetcache an answer with NSID maxanswersize -= EDNSOptionCodeSize + EDNSOptionLengthSize + mode_server_id.size(); } @@ -1665,7 +1724,7 @@ static void startDoResolve(void *p) checkFrameStreamExport(luaconfsLocal); #endif - DNSPacketWriter pw(packet, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass); + DNSPacketWriter pw(packet, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass, dc->d_mdp.d_header.opcode); pw.getHeader()->aa=0; pw.getHeader()->ra=1; @@ -1682,11 +1741,12 @@ static void startDoResolve(void *p) uint32_t minTTL = dc->d_ttlCap; SyncRes sr(dc->d_now); + sr.d_eventTrace = std::move(dc->d_eventTrace); sr.setId(MT->getTid()); bool DNSSECOK=false; - if(t_pdl) { - sr.setLuaEngine(t_pdl); + if(dc->d_luaContext) { + sr.setLuaEngine(dc->d_luaContext); } if(g_dnssecmode != DNSSECMode::Off) { sr.setDoDNSSEC(true); @@ -1750,7 +1810,7 @@ static void startDoResolve(void *p) dq.extendedErrorExtra = &dc->d_extendedErrorExtra; dq.meta = std::move(dc->d_meta); - if(ednsExtRCode != 0) { + if(ednsExtRCode != 0 || dc->d_mdp.d_header.opcode == Opcode::Notify) { goto sendit; } @@ -1779,8 +1839,8 @@ static void startDoResolve(void *p) sr.setCacheOnly(); } - if (t_pdl) { - t_pdl->prerpz(dq, res); + if (dc->d_luaContext) { + dc->d_luaContext->prerpz(dq, res, sr.d_eventTrace); } // Check if the client has a policy attached to it @@ -1827,7 +1887,7 @@ static void startDoResolve(void *p) } // if there is a RecursorLua active, and it 'took' the query in preResolve, we don't launch beginResolve - if (!t_pdl || !t_pdl->preresolve(dq, res)) { + if (!dc->d_luaContext || !dc->d_luaContext->preresolve(dq, res, sr.d_eventTrace)) { if (!g_dns64PrefixReverse.empty() && dq.qtype == QType::PTR && dq.qname.isPartOf(g_dns64PrefixReverse)) { res = getFakePTRRecords(dq.qname, ret); @@ -1838,7 +1898,7 @@ static void startDoResolve(void *p) if (wantsRPZ && appliedPolicy.d_kind != DNSFilterEngine::PolicyKind::NoAction) { - if (t_pdl && t_pdl->policyHitEventFilter(dc->d_source, dc->d_mdp.d_qname, QType(dc->d_mdp.d_qtype), dc->d_tcp, appliedPolicy, dc->d_policyTags, sr.d_discardedPolicies)) { + if (dc->d_luaContext && dc->d_luaContext->policyHitEventFilter(dc->d_source, dc->d_mdp.d_qname, QType(dc->d_mdp.d_qtype), dc->d_tcp, appliedPolicy, dc->d_policyTags, sr.d_discardedPolicies)) { /* reset to no match */ appliedPolicy = DNSFilterEngine::Policy(); } @@ -1913,10 +1973,10 @@ static void startDoResolve(void *p) } } - if (t_pdl || (g_dns64Prefix && dq.qtype == QType::AAAA && !vStateIsBogus(dq.validationState))) { + if (dc->d_luaContext || (g_dns64Prefix && dq.qtype == QType::AAAA && !vStateIsBogus(dq.validationState))) { if (res == RCode::NoError) { if (answerIsNOData(dc->d_mdp.d_qtype, res, ret)) { - if (t_pdl && t_pdl->nodata(dq, res)) { + if (dc->d_luaContext && dc->d_luaContext->nodata(dq, res, sr.d_eventTrace)) { shouldNotValidate = true; auto policyResult = handlePolicyHit(appliedPolicy, dc, sr, res, ret, pw); if (policyResult == PolicyResult::HaveAnswer) { @@ -1932,7 +1992,7 @@ static void startDoResolve(void *p) } } } - else if (res == RCode::NXDomain && t_pdl && t_pdl->nxdomain(dq, res)) { + else if (res == RCode::NXDomain && dc->d_luaContext && dc->d_luaContext->nxdomain(dq, res, sr.d_eventTrace)) { shouldNotValidate = true; auto policyResult = handlePolicyHit(appliedPolicy, dc, sr, res, ret, pw); if (policyResult == PolicyResult::HaveAnswer) { @@ -1943,7 +2003,7 @@ static void startDoResolve(void *p) } } - if (t_pdl && t_pdl->postresolve(dq, res)) { + if (dc->d_luaContext && dc->d_luaContext->postresolve(dq, res, sr.d_eventTrace)) { shouldNotValidate = true; auto policyResult = handlePolicyHit(appliedPolicy, dc, sr, res, ret, pw); // haveAnswer case redundant @@ -1953,7 +2013,7 @@ static void startDoResolve(void *p) } } } - else if (t_pdl) { + else if (dc->d_luaContext) { // preresolve returned true shouldNotValidate = true; auto policyResult = handlePolicyHit(appliedPolicy, dc, sr, res, ret, pw); @@ -2018,7 +2078,7 @@ static void startDoResolve(void *p) if(t_bogusremotes) t_bogusremotes->push_back(dc->d_source); if(t_bogusqueryring) - t_bogusqueryring->push_back(make_pair(dc->d_mdp.d_qname, dc->d_mdp.d_qtype)); + t_bogusqueryring->push_back({dc->d_mdp.d_qname, dc->d_mdp.d_qtype}); if(g_dnssecLogBogus || sr.doLog() || g_dnssecmode == DNSSECMode::ValidateForLog) { g_log<d_mdp.d_qname<<"|"<d_mdp.d_qtype)<getRemote()<<" validates as "<d_mdp.d_qname, dc->d_mdp.d_qtype, *i); + udr = udrCheckUniqueDNSRecord(nodlogger, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, *i); if (!hasUDR && udr) hasUDR = true; } @@ -2109,7 +2169,7 @@ static void startDoResolve(void *p) maxanswersize -= EDNSOptionCodeSize + EDNSOptionLengthSize + ecsPayload.size(); - returnedEdnsOptions.push_back(make_pair(EDNSOptionCode::ECS, std::move(ecsPayload))); + returnedEdnsOptions.emplace_back(EDNSOptionCode::ECS, std::move(ecsPayload)); } } @@ -2133,7 +2193,7 @@ static void startDoResolve(void *p) if (modulo > 0) { padSize = std::min(blockSize - modulo, remaining); } - returnedEdnsOptions.push_back(make_pair(EDNSOptionCode::PADDING, makeEDNSPaddingOptString(padSize))); + returnedEdnsOptions.emplace_back(EDNSOptionCode::PADDING, makeEDNSPaddingOptString(padSize)); } } @@ -2204,7 +2264,7 @@ static void startDoResolve(void *p) eee.extraText = std::move(extra); if (pw.size() < maxanswersize && (maxanswersize - pw.size()) >= (EDNSOptionCodeSize + EDNSOptionLengthSize + sizeof(eee.infoCode) + eee.extraText.size())) { - returnedEdnsOptions.push_back(make_pair(EDNSOptionCode::EXTENDEDERROR, makeEDNSExtendedErrorOptString(eee))); + returnedEdnsOptions.emplace_back(EDNSOptionCode::EXTENDEDERROR, makeEDNSExtendedErrorOptString(eee)); } } @@ -2223,11 +2283,16 @@ static void startDoResolve(void *p) #ifdef NOD_ENABLED bool nod = false; if (g_nodEnabled) { - if (nodCheckNewDomain(dc->d_mdp.d_qname)) { + if (nodCheckNewDomain(nodlogger, dc->d_mdp.d_qname)) { nod = true; } } #endif /* NOD_ENABLED */ + + if (variableAnswer || sr.wasVariable()) { + g_stats.variableResponses++; + } + if (t_protobufServers && !(luaconfsLocal->protobufExportConfig.taggedOnly && appliedPolicy.getName().empty() && dc->d_policyTags.empty())) { // Start constructing embedded DNSResponse object pbMessage.setResponseCode(pw.getHeader()->rcode); @@ -2251,6 +2316,49 @@ static void startDoResolve(void *p) // if (g_udrEnabled) ?? pbMessage.clearUDR(pbDataForCache->d_response); #endif + } + + if (!SyncRes::s_nopacketcache && !variableAnswer && !sr.wasVariable()) { + const auto& hdr = pw.getHeader(); + if ((hdr->rcode != RCode::NoError && hdr->rcode != RCode::NXDomain) || + (hdr->ancount == 0 && hdr->nscount == 0)) { + minTTL = min(minTTL, SyncRes::s_packetcacheservfailttl); + } + minTTL = min(minTTL, SyncRes::s_packetcachettl); + t_packetCache->insertResponsePacket(dc->d_tag, dc->d_qhash, std::move(dc->d_query), dc->d_mdp.d_qname, + dc->d_mdp.d_qtype, dc->d_mdp.d_qclass, + string((const char*)&*packet.begin(), packet.size()), + g_now.tv_sec, + minTTL, + dq.validationState, + std::move(pbDataForCache), dc->d_tcp); + } + if (!dc->d_tcp) { + struct msghdr msgh; + struct iovec iov; + cmsgbuf_aligned cbuf; + fillMSGHdr(&msgh, &iov, &cbuf, 0, (char*)&*packet.begin(), packet.size(), &dc->d_remote); + msgh.msg_control=NULL; + + if(g_fromtosockets.count(dc->d_socket)) { + addCMsgSrcAddr(&msgh, &cbuf, &dc->d_local, 0); + } + int sendErr = sendOnNBSocket(dc->d_socket, &msgh); + if (sendErr && g_logCommonErrors) { + g_log << Logger::Warning << "Sending UDP reply to client " << dc->getRemote() << " failed with: " + << strerror(sendErr) << endl; + } + + } + else { + bool hadError = sendResponseOverTCP(dc, packet); + finishTCPReply(dc, hadError, true); + } + + sr.d_eventTrace.add(RecEventTrace::AnswerSent); + + // Now do the per query changing part ot the protobuf message + if (t_protobufServers && !(luaconfsLocal->protobufExportConfig.taggedOnly && appliedPolicy.getName().empty() && dc->d_policyTags.empty())) { // Below are the fields that are not stored in the packet cache and will be appended here and on a cache hit if (g_useKernelTimestamp && dc->d_kernelTimestamp.tv_sec) { pbMessage.setQueryTime(dc->d_kernelTimestamp.tv_sec, dc->d_kernelTimestamp.tv_usec); @@ -2260,7 +2368,7 @@ static void startDoResolve(void *p) } pbMessage.setMessageIdentity(dc->d_uuid); pbMessage.setSocketFamily(dc->d_source.sin4.sin_family); - pbMessage.setSocketProtocol(dc->d_tcp); + pbMessage.setSocketProtocol(dc->d_tcp ? pdns::ProtoZero::Message::TransportProtocol::TCP : pdns::ProtoZero::Message::TransportProtocol::UDP); Netmask requestorNM(dc->d_source, dc->d_source.sin4.sin_family == AF_INET ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); ComboAddress requestor = requestorNM.getMaskedNetwork(); pbMessage.setFrom(requestor); @@ -2289,45 +2397,16 @@ static void startDoResolve(void *p) } } #endif /* NOD_ENABLED */ + if (sr.d_eventTrace.enabled() && SyncRes::s_event_trace_enabled & SyncRes::event_trace_to_pb) { + pbMessage.addEvents(sr.d_eventTrace); + } if (dc->d_logResponse) { protobufLogResponse(pbMessage); } } - if (variableAnswer || sr.wasVariable()) { - g_stats.variableResponses++; - } - if (!SyncRes::s_nopacketcache && !variableAnswer && !sr.wasVariable()) { - minTTL = min(minTTL, pw.getHeader()->rcode == RCode::ServFail ? SyncRes::s_packetcacheservfailttl : - SyncRes::s_packetcachettl); - t_packetCache->insertResponsePacket(dc->d_tag, dc->d_qhash, std::move(dc->d_query), dc->d_mdp.d_qname, - dc->d_mdp.d_qtype, dc->d_mdp.d_qclass, - string((const char*)&*packet.begin(), packet.size()), - g_now.tv_sec, - minTTL, - dq.validationState, - std::move(pbDataForCache), dc->d_tcp); - } - if (!dc->d_tcp) { - struct msghdr msgh; - struct iovec iov; - cmsgbuf_aligned cbuf; - fillMSGHdr(&msgh, &iov, &cbuf, 0, (char*)&*packet.begin(), packet.size(), &dc->d_remote); - msgh.msg_control=NULL; - - if(g_fromtosockets.count(dc->d_socket)) { - addCMsgSrcAddr(&msgh, &cbuf, &dc->d_local, 0); - } - int sendErr = sendOnNBSocket(dc->d_socket, &msgh); - if (sendErr && g_logCommonErrors) { - g_log << Logger::Warning << "Sending UDP reply to client " << dc->getRemote() << " failed with: " - << strerror(sendErr) << endl; - } - - } - else { - bool hadError = sendResponseOverTCP(dc, packet); - finishTCPReply(dc, hadError, true); + if (sr.d_eventTrace.enabled() && SyncRes::s_event_trace_enabled & SyncRes::event_trace_to_log) { + g_log << Logger::Info << sr.d_eventTrace.toString() << endl; } // Originally this code used a mix of floats, doubles, uint64_t with different units. @@ -2345,11 +2424,13 @@ static void startDoResolve(void *p) g_log<cacheMisses++; - } - else { - g_recCache->cacheHits++; + if (dc->d_mdp.d_header.opcode == Opcode::Query) { + if (sr.d_outqueries || sr.d_authzonequeries) { + g_recCache->cacheMisses++; + } + else { + g_recCache->cacheHits++; + } } g_stats.answers(spentUsec); @@ -2369,7 +2450,7 @@ static void startDoResolve(void *p) #ifdef NOD_ENABLED if (nod) { - sendNODLookup(dc->d_mdp.d_qname); + sendNODLookup(nodlogger, dc->d_mdp.d_qname); } #endif /* NOD_ENABLED */ @@ -2543,7 +2624,7 @@ static bool checkForCacheHit(bool qnameParsed, unsigned int tag, const string& d t_bogusremotes->push_back(source); } if (t_bogusqueryring) { - t_bogusqueryring->push_back(make_pair(qname, qtype)); + t_bogusqueryring->push_back({qname, qtype}); } } @@ -2566,6 +2647,27 @@ static bool checkForCacheHit(bool qnameParsed, unsigned int tag, const string& d return cacheHit; } +static void* pleaseWipeCaches(const DNSName& canon, bool subtree, uint16_t qtype) +{ + auto res = wipeCaches(canon, subtree, qtype); + g_log<func = [=]{ return pleaseWipeCaches(canon, true, 0xffff); }; + tmsg->wantAnswer = false; + if(write(s_threadInfos[0].pipes.writeToThread, &tmsg, sizeof(tmsg)) != sizeof(tmsg)) { + delete tmsg; + + unixDie("write to thread pipe returned wrong size or error"); + } +} + static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) { shared_ptr conn=boost::any_cast >(var); @@ -2691,7 +2793,7 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) conn->state = TCPConnection::BYTE0; std::unique_ptr dc; try { - dc=std::unique_ptr(new DNSComboWriter(conn->data, g_now)); + dc = std::make_unique(conn->data, g_now, t_pdl); } catch(const MOADNSException &mde) { g_stats.clientParseError++; @@ -2731,6 +2833,8 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) bool logQuery = false; bool qnameParsed = false; + dc->d_eventTrace.setEnabled(SyncRes::s_event_trace_enabled); + dc->d_eventTrace.add(RecEventTrace::ReqRecv); auto luaconfsLocal = g_luaconfs.getLocal(); if (checkProtobufExport(luaconfsLocal)) { needECS = true; @@ -2742,7 +2846,7 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) checkFrameStreamExport(luaconfsLocal); #endif - if(needECS || needXPF || (t_pdl && (t_pdl->d_gettag_ffi || t_pdl->d_gettag))) { + if(needECS || needXPF || (t_pdl && (t_pdl->d_gettag_ffi || t_pdl->d_gettag)) || dc->d_mdp.d_header.opcode == Opcode::Notify) { try { EDNSOptionViewMap ednsOptions; @@ -2758,10 +2862,14 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) try { if (t_pdl->d_gettag_ffi) { RecursorLua4::FFIParams params(qname, qtype, dc->d_destination, dc->d_source, dc->d_ednssubnet.source, dc->d_data, dc->d_policyTags, dc->d_records, ednsOptions, dc->d_proxyProtocolValues, requestorId, deviceId, deviceName, dc->d_routingTag, dc->d_rcode, dc->d_ttlCap, dc->d_variable, true, logQuery, dc->d_logResponse, dc->d_followCNAMERecords, dc->d_extendedErrorCode, dc->d_extendedErrorExtra, dc->d_responsePaddingDisabled, dc->d_meta); + dc->d_eventTrace.add(RecEventTrace::LuaGetTagFFI); dc->d_tag = t_pdl->gettag_ffi(params); + dc->d_eventTrace.add(RecEventTrace::LuaGetTagFFI, dc->d_tag, false); } else if (t_pdl->d_gettag) { + dc->d_eventTrace.add(RecEventTrace::LuaGetTag); dc->d_tag = t_pdl->gettag(dc->d_source, dc->d_ednssubnet.source, dc->d_destination, qname, qtype, &dc->d_policyTags, dc->d_data, ednsOptions, true, requestorId, deviceId, deviceName, dc->d_routingTag, dc->d_proxyProtocolValues); + dc->d_eventTrace.add(RecEventTrace::LuaGetTag, dc->d_tag, false); } } catch(const std::exception& e) { @@ -2807,7 +2915,8 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) } if (t_pdl) { - if (t_pdl->ipfilter(dc->d_source, dc->d_destination, *dh)) { + bool ipf = t_pdl->ipfilter(dc->d_source, dc->d_destination, *dh, dc->d_eventTrace); + if (ipf) { if (!g_quiet) { g_log<getTid()<<"/"<numProcesses()<<"] DROPPED TCP question from "<d_source.toStringWithPort()<<(dc->d_source != dc->d_remote ? " (via "+dc->d_remote.toStringWithPort()+")" : "")<<" based on policy"<d_mdp.d_header.opcode) { + if (dc->d_mdp.d_header.opcode != Opcode::Query && dc->d_mdp.d_header.opcode != Opcode::Notify) { g_stats.ignoredCount++; if (g_logCommonErrors) { - g_log<getRemote() <<" on server socket!"<d_mdp.d_header.opcode)<<" from TCP client "<< dc->getRemote() <<" on server socket!"<d_mdp.d_header.opcode == Opcode::Notify) { + if(!t_allowNotifyFrom || !t_allowNotifyFrom->match(dc->d_source)) { + if(!g_quiet) { + g_log<getTid()<<"] dropping TCP NOTIFY from "<d_source.toString()<<", address not matched by allow-notify-from"<getTid()<<"] dropping TCP NOTIFY from "<d_source.toString()<<", for "<d_tag, conn->data, qname, qtype, qclass, g_now, response, dc->d_qhash, pbData, true, dc->d_source); + if(dc->d_mdp.d_header.opcode == Opcode::Query) { + /* It might seem like a good idea to skip the packet cache lookup if we know that the answer is not cacheable, + but it means that the hash would not be computed. If some script decides at a later time to mark back the answer + as cacheable we would cache it with a wrong tag, so better safe than sorry. */ + dc->d_eventTrace.add(RecEventTrace::PCacheCheck); + bool cacheHit = checkForCacheHit(qnameParsed, dc->d_tag, conn->data, qname, qtype, qclass, g_now, response, dc->d_qhash, pbData, true, dc->d_source); + dc->d_eventTrace.add(RecEventTrace::PCacheCheck, cacheHit, false); + + if (cacheHit) { + if (!g_quiet) { + g_log<d_source.toStringWithPort()<<(dc->d_source != dc->d_remote ? " (via "+dc->d_remote.toStringWithPort()+")" : "")<d_logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) { - struct timeval tv{0, 0}; - protobufLogResponse(dh, luaconfsLocal, pbData, tv, true, dc->d_source, dc->d_destination, dc->d_ednssubnet, dc->d_uuid, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName, dc->d_meta); - } + bool hadError = sendResponseOverTCP(dc, response); + finishTCPReply(dc, hadError, false); + struct timeval now; + Utility::gettimeofday(&now, nullptr); + uint64_t spentUsec = uSec(now - start); + g_stats.cumulativeAnswers(spentUsec); + dc->d_eventTrace.add(RecEventTrace::AnswerSent); + + if (t_protobufServers && dc->d_logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) { + struct timeval tv{0, 0}; + protobufLogResponse(dh, luaconfsLocal, pbData, tv, true, dc->d_source, dc->d_destination, dc->d_ednssubnet, dc->d_uuid, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName, dc->d_meta, dc->d_eventTrace); + } + + if (dc->d_eventTrace.enabled() && SyncRes::s_event_trace_enabled & SyncRes::event_trace_to_log) { + g_log << Logger::Info << dc->d_eventTrace.toString() << endl; + } + return; + } // cache hit + } // query opcode + if(dc->d_mdp.d_header.opcode == Opcode::Notify) { if (!g_quiet) { - g_log<d_source.toStringWithPort()<<(dc->d_source != dc->d_remote ? " (via "+dc->d_remote.toStringWithPort()+")" : "")<d_source.toStringWithPort()<<(dc->d_source != dc->d_remote ? " (via "+dc->d_remote.toStringWithPort()+")" : "")<opcode, but we need to ensure that the response + // to this request does not get put into the packet cache + dc->d_variable = true; + } + + // setup for startDoResolve() in an mthread + ++conn->d_requestsInFlight; + if (conn->d_requestsInFlight >= TCPConnection::s_maxInFlight) { + t_fdm->removeReadFD(fd); // should no longer awake ourselves when there is data to read } else { - // No cache hit, setup for startDoResolve() in an mthread - ++conn->d_requestsInFlight; - if (conn->d_requestsInFlight >= TCPConnection::s_maxInFlight) { - t_fdm->removeReadFD(fd); // should no longer awake ourselves when there is data to read - } else { - Utility::gettimeofday(&g_now, nullptr); // needed? - struct timeval ttd = g_now; - t_fdm->setReadTTD(fd, ttd, g_tcpTimeout); - } - MT->makeThread(startDoResolve, dc.release()); // deletes dc - } // Cache hit or not + Utility::gettimeofday(&g_now, nullptr); // needed? + struct timeval ttd = g_now; + t_fdm->setReadTTD(fd, ttd, g_tcpTimeout); + } + MT->makeThread(startDoResolve, dc.release()); // deletes dc } // good query } // read full query } // reading query @@ -2965,7 +3120,7 @@ static void handleNewTCPQuestion(int fd, FDMultiplexer::funcparam_t& ) } } -static string* doProcessUDPQuestion(const std::string& question, const ComboAddress& fromaddr, const ComboAddress& destaddr, ComboAddress source, ComboAddress destination, struct timeval tv, int fd, std::vector& proxyProtocolValues) +static string* doProcessUDPQuestion(const std::string& question, const ComboAddress& fromaddr, const ComboAddress& destaddr, ComboAddress source, ComboAddress destination, struct timeval tv, int fd, std::vector& proxyProtocolValues, RecEventTrace& eventTrace) { gettimeofday(&g_now, nullptr); if (tv.tv_sec) { @@ -3021,8 +3176,8 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr bool variable = false; bool followCNAMEs = false; bool responsePaddingDisabled = false; + DNSName qname; try { - DNSName qname; uint16_t qtype=0; uint16_t qclass=0; bool qnameParsed=false; @@ -3038,7 +3193,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr */ #endif - if(needECS || needXPF || (t_pdl && (t_pdl->d_gettag || t_pdl->d_gettag_ffi))) { + if(needECS || needXPF || (t_pdl && (t_pdl->d_gettag || t_pdl->d_gettag_ffi)) || dh->opcode == Opcode::Notify) { try { EDNSOptionViewMap ednsOptions; bool xpfFound = false; @@ -3057,10 +3212,14 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr if (t_pdl->d_gettag_ffi) { RecursorLua4::FFIParams params(qname, qtype, destination, source, ednssubnet.source, data, policyTags, records, ednsOptions, proxyProtocolValues, requestorId, deviceId, deviceName, routingTag, rcode, ttlCap, variable, false, logQuery, logResponse, followCNAMEs, extendedErrorCode, extendedErrorExtra, responsePaddingDisabled, meta); + eventTrace.add(RecEventTrace::LuaGetTagFFI); ctag = t_pdl->gettag_ffi(params); + eventTrace.add(RecEventTrace::LuaGetTagFFI, ctag, false); } else if (t_pdl->d_gettag) { + eventTrace.add(RecEventTrace::LuaGetTag); ctag = t_pdl->gettag(source, ednssubnet.source, destination, qname, qtype, &policyTags, data, ednsOptions, false, requestorId, deviceId, deviceName, routingTag, proxyProtocolValues); + eventTrace.add(RecEventTrace::LuaGetTag, ctag, false); } } catch (const std::exception& e) { @@ -3088,38 +3247,48 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr if (ctag == 0 && !responsePaddingDisabled && g_paddingFrom.match(fromaddr)) { ctag = g_paddingTag; } - /* It might seem like a good idea to skip the packet cache lookup if we know that the answer is not cacheable, - but it means that the hash would not be computed. If some script decides at a later time to mark back the answer - as cacheable we would cache it with a wrong tag, so better safe than sorry. */ - bool cacheHit = checkForCacheHit(qnameParsed, ctag, question, qname, qtype, qclass, g_now, response, qhash, pbData, false, source); - if (cacheHit) { - if (t_protobufServers && logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) { - protobufLogResponse(dh, luaconfsLocal, pbData, tv, false, source, destination, ednssubnet, uniqueId, requestorId, deviceId, deviceName, meta); - } - if (!g_quiet) { - g_log<(&fromaddr)); - msgh.msg_control=NULL; + if(dh->opcode == Opcode::Query) { + /* It might seem like a good idea to skip the packet cache lookup if we know that the answer is not cacheable, + but it means that the hash would not be computed. If some script decides at a later time to mark back the answer + as cacheable we would cache it with a wrong tag, so better safe than sorry. */ + eventTrace.add(RecEventTrace::PCacheCheck); + bool cacheHit = checkForCacheHit(qnameParsed, ctag, question, qname, qtype, qclass, g_now, response, qhash, pbData, false, source); + eventTrace.add(RecEventTrace::PCacheCheck, cacheHit, false); + if (cacheHit) { + if (!g_quiet) { + g_log<(&fromaddr)); + msgh.msg_control=NULL; + + if(g_fromtosockets.count(fd)) { + addCMsgSrcAddr(&msgh, &cbuf, &destaddr, 0); + } + int sendErr = sendOnNBSocket(fd, &msgh); + eventTrace.add(RecEventTrace::AnswerSent); - if(g_fromtosockets.count(fd)) { - addCMsgSrcAddr(&msgh, &cbuf, &destaddr, 0); - } - int sendErr = sendOnNBSocket(fd, &msgh); - if (sendErr && g_logCommonErrors) { - g_log << Logger::Warning << "Sending UDP reply to client " << source.toStringWithPort() - << (source != fromaddr ? " (via " + fromaddr.toStringWithPort() + ")" : "") << " failed with: " - << strerror(sendErr) << endl; + if (t_protobufServers && logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) { + protobufLogResponse(dh, luaconfsLocal, pbData, tv, false, source, destination, ednssubnet, uniqueId, requestorId, deviceId, deviceName, meta, eventTrace); + } + + if (eventTrace.enabled() && SyncRes::s_event_trace_enabled & SyncRes::event_trace_to_log) { + g_log << Logger::Info << eventTrace.toString() << endl; + } + if (sendErr && g_logCommonErrors) { + g_log << Logger::Warning << "Sending UDP reply to client " << source.toStringWithPort() + << (source != fromaddr ? " (via " + fromaddr.toStringWithPort() + ")" : "") << " failed with: " + << strerror(sendErr) << endl; + } + struct timeval now; + Utility::gettimeofday(&now, nullptr); + uint64_t spentUsec = uSec(now - tv); + g_stats.cumulativeAnswers(spentUsec); + return 0; } - struct timeval now; - Utility::gettimeofday(&now, nullptr); - uint64_t spentUsec = uSec(now - tv); - g_stats.cumulativeAnswers(spentUsec); - return 0; } } catch (const std::exception& e) { @@ -3130,15 +3299,39 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr } if (t_pdl) { - if (t_pdl->ipfilter(source, destination, *dh)) { + bool ipf = t_pdl->ipfilter(source, destination, *dh, eventTrace); + if (ipf) { if (!g_quiet) { - g_log<getTid()<<"/"<numProcesses()<<"] DROPPED question from "<getTid()<<"/"<numProcesses()<<"] DROPPED question from "<opcode == Opcode::Notify) { + if(!isAllowNotifyForZone(qname)) { + if(!g_quiet) { + g_log<getTid()<<"] dropping UDP NOTIFY from "<opcode, but we need to ensure that the response + // to this request does not get put into the packet cache + variable = true; + } + if(MT->numProcesses() > g_maxMThreads) { if(!g_quiet) g_log<getTid()<<"/"<numProcesses()<<"] DROPPED question from "<(new DNSComboWriter(question, g_now, std::move(policyTags), std::move(data), std::move(records))); + auto dc = std::make_unique(question, g_now, std::move(policyTags), t_pdl, std::move(data), std::move(records)); dc->setSocket(fd); dc->d_tag=ctag; dc->d_qhash=qhash; @@ -3178,7 +3371,9 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr dc->d_responsePaddingDisabled = responsePaddingDisabled; dc->d_meta = std::move(meta); + dc->d_eventTrace = std::move(eventTrace); MT->makeThread(startDoResolve, (void*) dc.release()); // deletes dc + return 0; } @@ -3196,6 +3391,7 @@ static void handleNewUDPQuestion(int fd, FDMultiplexer::funcparam_t& var) cmsgbuf_aligned cbuf; bool firstQuery = true; std::vector proxyProtocolValues; + RecEventTrace eventTrace; for(size_t queriesCounter = 0; queriesCounter < s_maxUDPQueriesPerRound; queriesCounter++) { bool proxyProto = false; @@ -3205,6 +3401,9 @@ static void handleNewUDPQuestion(int fd, FDMultiplexer::funcparam_t& var) fillMSGHdr(&msgh, &iov, &cbuf, sizeof(cbuf), &data[0], data.size(), &fromaddr); if((len=recvmsg(fd, &msgh, 0)) >= 0) { + eventTrace.clear(); + eventTrace.setEnabled(SyncRes::s_event_trace_enabled); + eventTrace.add(RecEventTrace::ReqRecv); firstQuery = false; @@ -3290,10 +3489,10 @@ static void handleNewUDPQuestion(int fd, FDMultiplexer::funcparam_t& var) g_log<opcode) { + else if(dh->opcode != Opcode::Query && dh->opcode != Opcode::Notify) { g_stats.ignoredCount++; if(g_logCommonErrors) { - g_log<opcode<<" from "<opcode)<<" from "<qdcount == 0) { @@ -3303,6 +3502,17 @@ static void handleNewUDPQuestion(int fd, FDMultiplexer::funcparam_t& var) } } else { + if(dh->opcode == Opcode::Notify) { + if(!t_allowNotifyFrom || !t_allowNotifyFrom->match(&source)) { + if(!g_quiet) { + g_log<getTid()<<"] dropping UDP NOTIFY from "<& tcp setNonBlocking(fd); setSocketSendBuffer(fd, 65000); listen(fd, 128); - deferredAdds.push_back(make_pair(fd, handleNewTCPQuestion)); + deferredAdds.emplace_back(fd, handleNewTCPQuestion); tcpSockets.insert(fd); // we don't need to update g_listenSocketsAddresses since it doesn't work for TCP/IP: @@ -3581,7 +3792,7 @@ static void makeUDPServerSockets(deferredAdd_t& deferredAdds) setNonBlocking(fd); - deferredAdds.push_back(make_pair(fd, handleNewUDPQuestion)); + deferredAdds.emplace_back(fd, handleNewUDPQuestion); g_listenSocketsAddresses[fd]=sin; // this is written to only from the startup thread, not from the workers if(sin.sin4.sin_family == AF_INET) g_log<(pleaseGetEDNSStatusesSize)<(pleaseGetConcurrentQueries)<<" queries running, "<(pleaseGetConcurrentQueries)<<" queries running, "<(pleaseGetPacketCacheSize); uint64_t pcHits = broadcastAccFunction(pleaseGetPacketCacheHits); @@ -3734,6 +3945,7 @@ static void houseKeeping(void *) SyncRes::pruneThrottledServers(); SyncRes::pruneNonResolving(now.tv_sec - SyncRes::s_nonresolvingnsthrottletime); Utility::gettimeofday(&last_prune, nullptr); + t_tcp_manager.cleanup(now); } if(isHandlerThread()) { @@ -3748,10 +3960,10 @@ static void houseKeeping(void *) // Divide by 12 to get the original 2 hour cycle if s_maxcachettl is default (1 day) if (now.tv_sec - last_rootupdate > max(SyncRes::s_maxcachettl / 12, 10U)) { int res = SyncRes::getRootNS(g_now, nullptr, 0); - if (!res) { + if (res == 0) { last_rootupdate=now.tv_sec; try { - primeRootNSZones(g_dnssecmode != DNSSECMode::Off, 0); + primeRootNSZones(g_dnssecmode, 0); } catch (const std::exception& e) { g_log<(packet.data()), packet.length(), g_disthashseed); unsigned int target = selectWorker(hash); ThreadMSG* tmsg = new ThreadMSG(); @@ -4126,21 +4337,18 @@ template ThreadTimes broadcastAccFunction(const boost::function& static void handleRCC(int fd, FDMultiplexer::funcparam_t& var) { try { - string remote; - string msg = s_rcc.recv(&remote).d_str; + FDWrapper clientfd = accept(fd, nullptr, nullptr); + if (clientfd == -1) { + throw PDNSException("accept failed"); + } + string msg = s_rcc.recv(clientfd).d_str; + g_log << Logger::Info << "Received rec_control command '" << msg << "' via controlsocket" << endl; + RecursorControlParser rcp; RecursorControlParser::func_t* command; + auto answer = rcp.getAnswer(clientfd, msg, &command); - g_log << Logger::Info << "Received rec_control command '" << msg << "' from control socket" << endl; - auto answer = rcp.getAnswer(fd, msg, &command); - - // If we are inside a chroot, we need to strip - if (!arg()["chroot"].empty()) { - size_t len = arg()["chroot"].length(); - remote = remote.substr(len); - } - - s_rcc.send(answer, &remote); + s_rcc.send(clientfd, answer); command(); } catch(const std::exception& e) { @@ -4504,6 +4712,7 @@ static void checkOrFixFDS() { unsigned int availFDs=getFilenumLimit(); unsigned int wantFDs = g_maxMThreads * g_numWorkerThreads +25; // even healthier margin then before + wantFDs += g_numWorkerThreads * TCPOutConnectionManager::s_maxIdlePerThread; if(wantFDs > availFDs) { unsigned int hardlimit= getFilenumLimit(true); @@ -4512,7 +4721,7 @@ static void checkOrFixFDS() g_log< ng) +static void* pleaseSupplantAllowFrom(std::shared_ptr ng) { t_allowFrom = ng; return nullptr; } +static void* pleaseSupplantAllowNotifyFrom(std::shared_ptr ng) +{ + t_allowNotifyFrom = ng; + return nullptr; +} + +void* pleaseSupplantAllowNotifyFor(std::shared_ptr ns) +{ + t_allowNotifyFor = ns; + return nullptr; +} + int g_argc; char** g_argv; +static std::shared_ptr parseACL(const std::string& aclFile, const std::string& aclSetting) +{ + auto result = std::make_shared(); + + if(!::arg()[aclFile].empty()) { + string line; + ifstream ifs(::arg()[aclFile].c_str()); + if(!ifs) { + throw runtime_error("Could not open '"+::arg()[aclFile]+"': "+stringerror()); + } + + string::size_type pos; + while(getline(ifs,line)) { + pos=line.find('#'); + if(pos!=string::npos) + line.resize(pos); + boost::trim(line); + if(line.empty()) + continue; + + result->addMask(line); + } + g_log<size()<<" "< ips; + stringtok(ips, ::arg()[aclSetting], ", "); + + g_log<::const_iterator i = ips.begin(); i!= ips.end(); ++i) { + result->addMask(*i); + if(i!=ips.begin()) + g_log< oldAllowFrom = t_allowFrom; - std::shared_ptr allowFrom = std::make_shared(); - - if(!::arg()["allow-from-file"].empty()) { - string line; - ifstream ifs(::arg()["allow-from-file"].c_str()); - if(!ifs) { - throw runtime_error("Could not open '"+::arg()["allow-from-file"]+"': "+stringerror()); - } - - string::size_type pos; - while(getline(ifs,line)) { - pos=line.find('#'); - if(pos!=string::npos) - line.resize(pos); - boost::trim(line); - if(line.empty()) - continue; - allowFrom->addMask(line); - } - g_log<size() <<" allow-from ranges from file '"<<::arg()["allow-from-file"]<<"' - overriding 'allow-from' setting"< ips; - stringtok(ips, ::arg()["allow-from"], ", "); - g_log<::const_iterator i = ips.begin(); i!= ips.end(); ++i) { - allowFrom->addMask(*i); - if(i!=ips.begin()) - g_log<size() == 0) { if(::arg()["local-address"]!="127.0.0.1" && ::arg().asNum("local-port")==53) g_log<doWipePacketCache(canon, qtype, subtree)); +} + +struct WipeCacheResult wipeCaches(const DNSName& canon, bool subtree, uint16_t qtype) +{ + struct WipeCacheResult res; + + try { + res.record_count = g_recCache->doWipeCache(canon, subtree, qtype); + res.packet_count = broadcastAccFunction([=]{ return pleaseWipePacketCache(canon, subtree, qtype);}); + res.negative_record_count = g_negCache->wipe(canon, subtree); + if (g_aggressiveNSECCache) { + g_aggressiveNSECCache->removeZoneInfo(canon, subtree); + } + } + catch (const std::exception& e) { + g_log<(millis) % 1000) * 1000 }; + TCPOutConnectionManager::s_maxIdlePerAuth = ::arg().asNum("tcp-out-max-idle-per-auth"); + TCPOutConnectionManager::s_maxQueries = ::arg().asNum("tcp-out-max-queries"); + TCPOutConnectionManager::s_maxIdlePerThread = ::arg().asNum("tcp-out-max-idle-per-thread"); + g_gettagNeedsEDNSOptions = ::arg().mustDo("gettag-needs-edns-options"); g_statisticsInterval = ::arg().asNum("statistics-interval"); @@ -5123,7 +5401,7 @@ static int serviceMain(int argc, char*argv[]) { CarbonConfig config; stringtok(config.servers, arg()["carbon-server"], ", "); - config.hostname = arg()["carbon-server"]; + config.hostname = arg()["carbon-ourname"]; config.instance_name = arg()["carbon-instance"]; config.namespace_name = arg()["carbon-namespace"]; g_carbonConfig.setState(std::move(config)); @@ -5289,6 +5567,8 @@ static int serviceMain(int argc, char*argv[]) } startLuaConfigDelayedThreads(delayedLuaThreads, g_luaconfs.getCopy().generation); + delayedLuaThreads.rpzPrimaryThreads.clear(); // no longer needed + delayedLuaThreads.ztcConfigs.clear(); // no longer needed makeThreadPipes(); @@ -5432,8 +5712,10 @@ try SyncRes tmp(g_now); // make sure it allocates tsstorage before we do anything, like primeHints or so.. SyncRes::setDomainMap(g_initialDomainMap); t_allowFrom = g_initialAllowFrom; - t_udpclientsocks = std::unique_ptr(new UDPClientSocks()); - t_tcpClientCounts = std::unique_ptr(new tcpClientCounts_t()); + t_allowNotifyFrom = g_initialAllowNotifyFrom; + t_allowNotifyFor = g_initialAllowNotifyFor; + t_udpclientsocks = std::make_unique(); + t_tcpClientCounts = std::make_unique(); if (threadInfo.isHandler) { if (!primeHints()) { @@ -5442,11 +5724,10 @@ try g_log<(new RecursorPacketCache()); - + t_packetCache = std::make_unique(); #ifdef NOD_ENABLED if (threadInfo.isWorker) @@ -5470,25 +5751,25 @@ try unsigned int ringsize=::arg().asNum("stats-ringbuffer-entries") / g_numWorkerThreads; if(ringsize) { - t_remotes = std::unique_ptr(new addrringbuf_t()); + t_remotes = std::make_unique(); if(g_weDistributeQueries) t_remotes->set_capacity(::arg().asNum("stats-ringbuffer-entries") / g_numDistributorThreads); else t_remotes->set_capacity(ringsize); - t_servfailremotes = std::unique_ptr(new addrringbuf_t()); + t_servfailremotes = std::make_unique(); t_servfailremotes->set_capacity(ringsize); - t_bogusremotes = std::unique_ptr(new addrringbuf_t()); + t_bogusremotes = std::make_unique(); t_bogusremotes->set_capacity(ringsize); - t_largeanswerremotes = std::unique_ptr(new addrringbuf_t()); + t_largeanswerremotes = std::make_unique(); t_largeanswerremotes->set_capacity(ringsize); - t_timeouts = std::unique_ptr(new addrringbuf_t()); + t_timeouts = std::make_unique(); t_timeouts->set_capacity(ringsize); - t_queryring = std::unique_ptr > >(new boost::circular_buffer >()); + t_queryring = std::make_unique>>(); t_queryring->set_capacity(ringsize); - t_servfailqueryring = std::unique_ptr > >(new boost::circular_buffer >()); + t_servfailqueryring = std::make_unique>>(); t_servfailqueryring->set_capacity(ringsize); - t_bogusqueryring = std::unique_ptr > >(new boost::circular_buffer >()); + t_bogusqueryring = std::make_unique>>(); t_bogusqueryring->set_capacity(ringsize); } MT = std::make_unique(::arg().asNum("stack-size")); @@ -5507,7 +5788,9 @@ try t_fdm=getMultiplexer(); RecursorWebServer *rws = nullptr; - + + t_fdm->addReadFD(threadInfo.pipes.readToThread, handlePipeRequest); + if(threadInfo.isHandler) { if(::arg().mustDo("webserver")) { g_log<getName() << "' multiplexer"<addReadFD(threadInfo.pipes.readToThread, handlePipeRequest); t_fdm->addReadFD(threadInfo.pipes.readQueriesToThread, handlePipeRequest); if (threadInfo.isListener) { @@ -5746,6 +6027,7 @@ int main(int argc, char **argv) ::arg().set("webserver-password", "Password required for accessing the webserver") = ""; ::arg().set("webserver-allow-from","Webserver access is only allowed from these subnets")="127.0.0.1,::1"; ::arg().set("webserver-loglevel", "Amount of logging in the webserver (none, normal, detailed)") = "normal"; + ::arg().setSwitch("webserver-hash-plaintext-credentials","Whether to hash passwords and api keys supplied in plaintext, to prevent keeping the plaintext version in memory at runtime")="no"; ::arg().set("carbon-ourname", "If set, overrides our reported hostname for carbon stats")=""; ::arg().set("carbon-server", "If set, send metrics in carbon (graphite) format to this server IP address")=""; ::arg().set("carbon-interval", "Number of seconds between carbon (graphite) updates")="30"; @@ -5795,6 +6077,10 @@ int main(int argc, char **argv) ::arg().set("version-string", "string reported on version.pdns or version.bind")=fullVersionString(); ::arg().set("allow-from", "If set, only allow these comma separated netmasks to recurse")=LOCAL_NETS; ::arg().set("allow-from-file", "If set, load allowed netmasks from this file")=""; + ::arg().set("allow-notify-for", "If set, NOTIFY requests for these zones will be allowed")=""; + ::arg().set("allow-notify-for-file", "If set, load NOTIFY-allowed zones from this file")=""; + ::arg().set("allow-notify-from", "If set, NOTIFY requests from these comma separated netmasks will be allowed")=""; + ::arg().set("allow-notify-from-file", "If set, load NOTIFY-allowed netmasks from this file")=""; ::arg().set("entropy-source", "If set, read entropy from this file")="/dev/urandom"; ::arg().set("dont-query", "If set, do not query these netmasks for DNS data")=DONT_QUERY; ::arg().set("max-tcp-per-client", "If set, maximum number of TCP sessions per client (IP address)")="0"; @@ -5861,7 +6147,7 @@ int main(int argc, char **argv) for (size_t idx = 0; idx < 128; idx++) { defaultAPIDisabledStats += ", ecs-v6-response-bits-" + std::to_string(idx + 1); } - std::string defaultDisabledStats = defaultAPIDisabledStats + ", cumul-answers, cumul-auth4answers, cumul-auth6answers, policy-hits"; + std::string defaultDisabledStats = defaultAPIDisabledStats + ", cumul-clientanswers, cumul-authanswers, policy-hits"; ::arg().set("stats-api-blacklist", "List of statistics that are disabled when retrieving the complete list of statistics via the API (deprecated)")=defaultAPIDisabledStats; ::arg().set("stats-carbon-blacklist", "List of statistics that are prevented from being exported via Carbon (deprecated)")=defaultDisabledStats; @@ -5899,6 +6185,7 @@ int main(int argc, char **argv) ::arg().setSwitch("qname-minimization", "Use Query Name Minimization")="yes"; ::arg().setSwitch("nothing-below-nxdomain", "When an NXDOMAIN exists in cache for a name with fewer labels than the qname, send NXDOMAIN without doing a lookup (see RFC 8020)")="dnssec"; ::arg().set("max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0"; + ::arg().set("max-include-depth", "Maximum nested $INCLUDE depth when loading a zone from a file")="20"; ::arg().set("record-cache-shards", "Number of shards in the record cache")="1024"; ::arg().set("refresh-on-ttl-perc", "If a record is requested from the cache and only this % of original TTL remains, refetch") = "0"; @@ -5930,6 +6217,13 @@ int main(int argc, char **argv) ::arg().setSwitch("dot-to-port-853", "Force DoT connection to target port 853 if DoT compiled in")="yes"; ::arg().set("dot-to-auth-names", "Use DoT to authoritative servers with these names or suffixes")=""; + ::arg().set("event-trace-enabled", "If set, event traces are collected and send out via protobuf logging (1), logfile (2) or both(3)")="0"; + + ::arg().set("tcp-out-max-idle-ms", "Time TCP/DoT connections are left idle in milliseconds or 0 if no limit") = "10000"; + ::arg().set("tcp-out-max-idle-per-auth", "Maximum number of idle TCP/DoT connections to a specific IP per thread, 0 means do not keep idle connections open") = "10"; + ::arg().set("tcp-out-max-queries", "Maximum total number of queries per TCP/DoT connection, 0 means no limit") = "0"; + ::arg().set("tcp-out-max-idle-per-thread", "Maximum number of idle TCP/DoT connections per thread") = "100"; + ::arg().setSwitch("structured-logging", "Prefer structured logging") = "yes"; ::arg().setCmd("help","Provide a helpful message"); ::arg().setCmd("version","Print version string"); @@ -5978,13 +6272,15 @@ int main(int argc, char **argv) auto startupLog = g_slog->withName("startup"); if(!::arg().file(configname.c_str())) { - startupLog->error("No such file", "Unable to parse configuration file", "config_file", Logging::Loggable(configname)); + SLOG(g_log<error("No such file", "Unable to parse configuration file", "config_file", Logging::Loggable(configname))); } ::arg().parse(argc,argv); if( !::arg()["chroot"].empty() && !::arg()["api-config-dir"].empty() ) { - startupLog->info("Cannot use chroot and enable the API at the same time"); + SLOG(g_log<info("Cannot use chroot and enable the API at the same time")); exit(EXIT_FAILURE); } @@ -5997,13 +6293,15 @@ int main(int argc, char **argv) if(::arg().asNum("threads")==1) { if (::arg().mustDo("pdns-distributes-queries")) { - startupLog->v(1)->info("Only one thread, no need to distribute queries ourselves"); + SLOG(g_log<v(1)->info("Only one thread, no need to distribute queries ourselves")); ::arg().set("pdns-distributes-queries")="no"; } } if(::arg().mustDo("pdns-distributes-queries") && ::arg().asNum("distributor-threads") <= 0) { - startupLog->v(1)->info("Asked to run with pdns-distributes-queries set but no distributor threads, raising to 1"); + SLOG(g_log<v(1)->info("Asked to run with pdns-distributes-queries set but no distributor threads, raising to 1")); ::arg().set("distributor-threads")="1"; } @@ -6016,8 +6314,8 @@ int main(int argc, char **argv) cout<<::arg().helpstring(::arg()["help"])<(new MemRecursorCache(::arg().asNum("record-cache-shards"))); - g_negCache = std::unique_ptr(new NegCache(::arg().asNum("record-cache-shards"))); + g_recCache = std::make_unique(::arg().asNum("record-cache-shards")); + g_negCache = std::make_unique(::arg().asNum("record-cache-shards")); g_quiet=::arg().mustDo("quiet"); Logger::Urgency logUrgency = (Logger::Urgency)::arg().asNum("loglevel"); diff --git a/pdns/pdnsutil.cc b/pdns/pdnsutil.cc index de5503f37d35..13a5bfb80ff8 100644 --- a/pdns/pdnsutil.cc +++ b/pdns/pdnsutil.cc @@ -5,6 +5,7 @@ #include +#include "credentials.hh" #include "dnsseckeeper.hh" #include "dnssecinfra.hh" #include "statbag.hh" @@ -107,6 +108,7 @@ static void loadMainConfig(const std::string& configdir) ::arg().set("max-signature-cache-entries", "Maximum number of signatures cache entries")=""; ::arg().set("rng", "Specify random number generator to use. Valid values are auto,sodium,openssl,getrandom,arc4random,urandom.")="auto"; ::arg().set("max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0"; + ::arg().set("max-include-depth", "Maximum nested $INCLUDE depth when loading a zone from a file")="20"; ::arg().setSwitch("upgrade-unknown-types","Transparently upgrade known TYPExxx records. Recommended to keep off, except for PowerDNS upgrades until data sources are cleaned up")="no"; ::arg().laxFile(configname.c_str()); @@ -229,7 +231,7 @@ static bool rectifyAllZones(DNSSECKeeper &dk, bool quiet = false) vector domainInfo; bool result = true; - B.getAllDomains(&domainInfo); + B.getAllDomains(&domainInfo, false, false); for(const DomainInfo& di : domainInfo) { if (!quiet) { cerr<<"Rectifying "<= 2) { + if(parts[1].find('@') != string::npos) { + cout<<"[Warning] Found @-sign in SOA RNAME, should probably be a dot (.): "<(); auto& seenIds = seenInfos.get<1>(); - B.getAllDomains(&domainInfo, true); + B.getAllDomains(&domainInfo, true, true); int errors=0; for(auto di : domainInfo) { if (checkZone(dk, B, di.zone) > 0) { @@ -1030,7 +1055,7 @@ static int listKeys(const string &zname, DNSSECKeeper& dk){ listKey(di, dk); } else { vector domainInfo; - B.getAllDomains(&domainInfo, g_verbose); + B.getAllDomains(&domainInfo, false, g_verbose); bool printHeader = true; for (const auto& di : domainInfo) { listKey(di, dk, printHeader); @@ -1173,6 +1198,7 @@ static int editZone(const DNSName &zone) { cmdline.clear(); ZoneParserTNG zpt(tmpnam, g_rootdnsname); zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps")); + zpt.setMaxIncludes(::arg().asNum("max-include-depth")); DNSResourceRecord zrr; map, vector > grouped; try { @@ -1635,7 +1661,7 @@ static int listAllZones(const string &type="") { UeberBackend B("default"); vector domains; - B.getAllDomains(&domains, g_verbose); + B.getAllDomains(&domains, false, g_verbose); int count = 0; for (const auto& di: domains) { @@ -1827,7 +1853,7 @@ static bool showZone(DNSSECKeeper& dk, const DNSName& zone, bool exportDS = fals } } else if(di.kind == DomainInfo::Slave) { - cout << "Primary" << addS(di.masters) << ": "; + cout << "Primar" << addS(di.masters, "y", "ies") << ": "; for(const auto& m : di.masters) cout< 1) { + try { + workFactor = pdns_stou(cmds.at(1)); + } + catch (const std::exception& e) { + cerr<<"Unable to parse the supplied work factor: "< domainInfo; - B.getAllDomains(&domainInfo); + B.getAllDomains(&domainInfo, false, false); unsigned int zonesSecured=0, zoneErrors=0; for(const DomainInfo& di : domainInfo) { @@ -3071,6 +3121,14 @@ try } return 0; } + else if(cmds.at(0) == "hash-password") { + if (cmds.size() < 2) { + cerr<<"Syntax: pdnsutil hash-password PASSWORD"< domains; - tgt->getAllDomains(&domains, true); + tgt->getAllDomains(&domains, false, true); if (domains.size()>0) throw PDNSException("Target backend has zone(s), please clean it first"); - src->getAllDomains(&domains, true); + src->getAllDomains(&domains, false, true); // iterate zones for(const DomainInfo& di: domains) { size_t nr,nc,nm,nk; diff --git a/pdns/pkcs11signers.cc b/pdns/pkcs11signers.cc index d74bb66e4dd5..3024fc435a72 100644 --- a/pdns/pkcs11signers.cc +++ b/pdns/pkcs11signers.cc @@ -7,8 +7,6 @@ #include #include -#include - #include "pdns/dnssecinfra.hh" #include "pdns/logger.hh" #include "pdns/pdnsexception.hh" @@ -153,7 +151,7 @@ class P11KitAttribute { // this bit is used for getting attribute from object // we provide a pointer for GetAttributeValue to write to CK_BYTE_PTR allocate(CK_ULONG amount) { - buffer = std::unique_ptr(new unsigned char[amount]); + buffer = std::make_unique(amount); buflen = amount; return buffer.get(); } @@ -218,7 +216,6 @@ class Pkcs11Slot { CK_SESSION_HANDLE d_session; CK_SLOT_ID d_slot; CK_RV d_err; - std::mutex d_m; void logError(const std::string& operation) const { if (d_err) { @@ -235,7 +232,6 @@ class Pkcs11Slot { d_err(0) { CK_TOKEN_INFO tokenInfo; - std::lock_guard l(d_m); if ((d_err = d_functions->C_OpenSession(this->d_slot, CKF_SERIAL_SESSION|CKF_RW_SESSION, 0, 0, &(this->d_session)))) { logError("C_OpenSession"); @@ -253,7 +249,7 @@ class Pkcs11Slot { bool Login(const std::string& pin) { if (d_logged_in) return true; - std::unique_ptr uPin(new unsigned char[pin.size()]); + auto uPin = std::make_unique(pin.size()); memcpy(uPin.get(), pin.c_str(), pin.size()); d_err = d_functions->C_Login(this->d_session, CKU_USER, uPin.get(), pin.size()); memset(uPin.get(), 0, pin.size()); @@ -272,15 +268,13 @@ class Pkcs11Slot { CK_FUNCTION_LIST* f() { return d_functions; } - std::mutex& m() { return d_m; } - - static std::shared_ptr GetSlot(const std::string& module, const string& tokenId); + static std::shared_ptr> GetSlot(const std::string& module, const string& tokenId); static CK_RV HuntSlot(const string& tokenId, CK_SLOT_ID &slotId, _CK_SLOT_INFO* info, CK_FUNCTION_LIST* functions); }; class Pkcs11Token { private: - std::shared_ptr d_slot; + std::shared_ptr> d_slot; CK_OBJECT_HANDLE d_public_key; CK_OBJECT_HANDLE d_private_key; @@ -333,12 +327,12 @@ class Pkcs11Token { } public: - Pkcs11Token(const std::shared_ptr& slot, const std::string& label, const std::string& pub_label); + Pkcs11Token(const std::shared_ptr>& slot, const std::string& label, const std::string& pub_label); ~Pkcs11Token(); bool Login(const std::string& pin) { if (pin.empty()) return false; // no empty pin. - if (d_slot->Login(pin) == true) { + if (d_slot->lock()->Login(pin) == true) { LoadAttributes(); } @@ -346,20 +340,20 @@ class Pkcs11Token { } bool LoggedIn() { - if (d_loaded == false && d_slot->LoggedIn() == true) { + if (d_loaded == false && d_slot->lock()->LoggedIn() == true) { LoadAttributes(); } - return d_slot->LoggedIn(); + return d_slot->lock()->LoggedIn(); } void LoadAttributes() { - std::lock_guard l(d_slot->m()); + auto slot = d_slot->lock(); std::vector attr; std::vector key; attr.push_back(P11KitAttribute(CKA_CLASS, (unsigned long)CKO_PRIVATE_KEY)); // attr.push_back(P11KitAttribute(CKA_SIGN, (char)CK_TRUE)); attr.push_back(P11KitAttribute(CKA_LABEL, d_label)); - FindObjects2(attr, key, 1); + FindObjects2(*slot, attr, key, 1); if (key.size() == 0) { g_log<& pubAttributes, std::vector& privAttributes, CK_OBJECT_HANDLE_PTR pubKey, CK_OBJECT_HANDLE_PTR privKey) { { - std::lock_guard l(d_slot->m()); + auto slot = d_slot->lock(); size_t k; - std::unique_ptr pubAttr(new CK_ATTRIBUTE[pubAttributes.size()]); - std::unique_ptr privAttr(new CK_ATTRIBUTE[privAttributes.size()]); + auto pubAttr = std::make_unique(pubAttributes.size()); + auto privAttr = std::make_unique(pubAttributes.size()); k = 0; for(P11KitAttribute& attribute : pubAttributes) { @@ -436,7 +430,7 @@ class Pkcs11Token { k++; } - d_err = this->d_slot->f()->C_GenerateKeyPair(d_slot->Session(), mechanism, pubAttr.get(), pubAttributes.size(), privAttr.get(), privAttributes.size(), pubKey, privKey); + d_err = slot->f()->C_GenerateKeyPair(slot->Session(), mechanism, pubAttr.get(), pubAttributes.size(), privAttr.get(), privAttributes.size(), pubKey, privKey); logError("C_GenerateKeyPair"); } @@ -446,14 +440,13 @@ class Pkcs11Token { } int Sign(const std::string& data, std::string& result, CK_MECHANISM_PTR mechanism) { - std::lock_guard l(d_slot->m()); - CK_BYTE buffer[1024]; CK_ULONG buflen = sizeof buffer; // should be enough for most signatures. + auto slot = d_slot->lock(); // perform signature - if ((d_err = this->d_slot->f()->C_SignInit(d_slot->Session(), mechanism, d_private_key))) { logError("C_SignInit"); return d_err; } - d_err = this->d_slot->f()->C_Sign(d_slot->Session(), (unsigned char*)data.c_str(), data.size(), buffer, &buflen); + if ((d_err = slot->f()->C_SignInit(slot->Session(), mechanism, d_private_key))) { logError("C_SignInit"); return d_err; } + d_err = slot->f()->C_Sign(slot->Session(), (unsigned char*)data.c_str(), data.size(), buffer, &buflen); if (!d_err) { result.assign((char*)buffer, buflen); @@ -465,21 +458,21 @@ class Pkcs11Token { } int Verify(const std::string& data, const std::string& signature, CK_MECHANISM_PTR mechanism) { - std::lock_guard l(d_slot->m()); + auto slot = d_slot->lock(); - if ((d_err = this->d_slot->f()->C_VerifyInit(d_slot->Session(), mechanism, d_public_key))) { logError("C_VerifyInit"); return d_err; } - d_err = this->d_slot->f()->C_Verify(d_slot->Session(), (unsigned char*)data.c_str(), data.size(), (unsigned char*)signature.c_str(), signature.size()); + if ((d_err = slot->f()->C_VerifyInit(slot->Session(), mechanism, d_public_key))) { logError("C_VerifyInit"); return d_err; } + d_err = slot->f()->C_Verify(slot->Session(), (unsigned char*)data.c_str(), data.size(), (unsigned char*)signature.c_str(), signature.size()); logError("C_Verify"); return d_err; } int Digest(const std::string& data, std::string& result, CK_MECHANISM_PTR mechanism) { - std::lock_guard l(d_slot->m()); - CK_BYTE buffer[1024]; CK_ULONG buflen = sizeof buffer; // should be enough for most digests - if ((d_err = this->d_slot->f()->C_DigestInit(d_slot->Session(), mechanism))) { logError("C_DigestInit"); return d_err; } - d_err = this->d_slot->f()->C_Digest(d_slot->Session(), (unsigned char*)data.c_str(), data.size(), buffer, &buflen); + + auto slot = d_slot->lock(); + if ((d_err = slot->f()->C_DigestInit(slot->Session(), mechanism))) { logError("C_DigestInit"); return d_err; } + d_err = slot->f()->C_Digest(slot->Session(), (unsigned char*)data.c_str(), data.size(), buffer, &buflen); if (!d_err) { result.assign((char*)buffer, buflen); } @@ -488,41 +481,42 @@ class Pkcs11Token { return d_err; } - int DigestInit(CK_MECHANISM_PTR mechanism) { - d_err = d_slot->f()->C_DigestInit(d_slot->Session(), mechanism); + int DigestInit(Pkcs11Slot& slot, CK_MECHANISM_PTR mechanism) { + d_err = slot.f()->C_DigestInit(slot.Session(), mechanism); logError("C_DigestInit"); return d_err; } - int DigestUpdate(const std::string& data) { - d_err = d_slot->f()->C_DigestUpdate(d_slot->Session(), (unsigned char*)data.c_str(), data.size()); + int DigestUpdate(Pkcs11Slot& slot, const std::string& data) { + d_err = slot.f()->C_DigestUpdate(slot.Session(), (unsigned char*)data.c_str(), data.size()); logError("C_DigestUpdate"); return d_err; } int DigestKey(std::string& result) { - std::lock_guard l(d_slot->m()); + auto slot = d_slot->lock(); CK_MECHANISM mech; mech.mechanism = CKM_SHA_1; - DigestInit(&mech); + DigestInit(*slot, &mech); if (d_key_type == CKK_RSA) { - DigestUpdate(d_modulus); - DigestUpdate(d_exponent); + DigestUpdate(*slot, d_modulus); + DigestUpdate(*slot, d_exponent); } else if (d_key_type == CKK_EC || d_key_type == CKK_ECDSA) { - DigestUpdate(d_ec_point); + DigestUpdate(*slot, d_ec_point); } - DigestFinal(result); + DigestFinal(*slot, result); return d_err; } - int DigestFinal(std::string& result) { + int DigestFinal(Pkcs11Slot& slot, std::string& result) { CK_BYTE buffer[1024] = {0}; CK_ULONG buflen = sizeof buffer; // should be enough for most digests - d_err = d_slot->f()->C_DigestFinal(d_slot->Session(), buffer, &buflen); + + d_err = slot.f()->C_DigestFinal(slot.Session(), buffer, &buflen); if (!d_err) { result.assign((char*)buffer, buflen); } @@ -531,18 +525,13 @@ class Pkcs11Token { return d_err; } - int FindObjects(const std::vector& attributes, std::vector& objects, int maxobjects) { - std::lock_guard l(d_slot->m()); - return FindObjects2(attributes, objects, maxobjects); - } - - int FindObjects2(const std::vector& attributes, std::vector& objects, int maxobjects) { + int FindObjects2(Pkcs11Slot& slot, const std::vector& attributes, std::vector& objects, int maxobjects) { CK_RV rv; size_t k; unsigned long count; - std::unique_ptr handles(new CK_OBJECT_HANDLE[maxobjects]); - std::unique_ptr attr(new CK_ATTRIBUTE[attributes.size()]); + auto handles = std::make_unique(maxobjects); + auto attr = std::make_unique(attributes.size()); k = 0; for(const P11KitAttribute& attribute : attributes) { @@ -551,7 +540,7 @@ class Pkcs11Token { } // perform search - d_err = this->d_slot->f()->C_FindObjectsInit(d_slot->Session(), attr.get(), k); + d_err = slot.f()->C_FindObjectsInit(slot.Session(), attr.get(), k); if (d_err) { logError("C_FindObjectsInit"); @@ -559,7 +548,7 @@ class Pkcs11Token { } count = maxobjects; - rv = d_err = this->d_slot->f()->C_FindObjects(d_slot->Session(), handles.get(), maxobjects, &count); + rv = d_err = slot.f()->C_FindObjects(slot.Session(), handles.get(), maxobjects, &count); objects.clear(); if (!rv) { @@ -570,22 +559,16 @@ class Pkcs11Token { logError("C_FindObjects"); - d_err = this->d_slot->f()->C_FindObjectsFinal(d_slot->Session()); + d_err = slot.f()->C_FindObjectsFinal(slot.Session()); logError("C_FindObjectsFinal"); return rv; } - int GetAttributeValue(const CK_OBJECT_HANDLE& object, std::vector& attributes) - { - std::lock_guard l(d_slot->m()); - return GetAttributeValue2(object, attributes); - } - - int GetAttributeValue2(const CK_OBJECT_HANDLE& object, std::vector& attributes) + int GetAttributeValue2(Pkcs11Slot& slot, const CK_OBJECT_HANDLE& object, std::vector& attributes) { size_t k; - std::unique_ptr attr(new CK_ATTRIBUTE[attributes.size()]); + auto attr = std::make_unique(attributes.size()); k = 0; for(P11KitAttribute &attribute : attributes) { @@ -594,7 +577,7 @@ class Pkcs11Token { } // round 1 - get attribute sizes - d_err = d_slot->f()->C_GetAttributeValue(d_slot->Session(), object, attr.get(), attributes.size()); + d_err = slot.f()->C_GetAttributeValue(slot.Session(), object, attr.get(), attributes.size()); logError("C_GetAttributeValue"); if (d_err) { return d_err; @@ -608,7 +591,7 @@ class Pkcs11Token { } // round 2 - get actual values - d_err = d_slot->f()->C_GetAttributeValue(d_slot->Session(), object, attr.get(), attributes.size()); + d_err = slot.f()->C_GetAttributeValue(slot.Session(), object, attr.get(), attributes.size()); logError("C_GetAttributeValue"); // copy values to map and release allocated memory @@ -648,7 +631,7 @@ class Pkcs11Token { static std::shared_ptr GetToken(const std::string& module, const string& tokenId, const std::string& label, const std::string& pub_label); }; -static std::map > pkcs11_slots; +static std::map > > pkcs11_slots; static std::map > pkcs11_tokens; CK_RV Pkcs11Slot::HuntSlot(const string& tokenId, CK_SLOT_ID &slotId, _CK_SLOT_INFO* info, CK_FUNCTION_LIST* functions) @@ -712,12 +695,12 @@ CK_RV Pkcs11Slot::HuntSlot(const string& tokenId, CK_SLOT_ID &slotId, _CK_SLOT_I return CKR_SLOT_ID_INVALID; } -std::shared_ptr Pkcs11Slot::GetSlot(const std::string& module, const string& tokenId) { +std::shared_ptr> Pkcs11Slot::GetSlot(const std::string& module, const string& tokenId) { // see if we can find module std::string sidx = module; sidx.append("|"); sidx.append(tokenId); - std::map >::iterator slotIter; + std::map > >::iterator slotIter; CK_RV err; CK_FUNCTION_LIST* functions; @@ -731,8 +714,8 @@ std::shared_ptr Pkcs11Slot::GetSlot(const std::string& module, const #else functions = p11_kit_registered_name_to_module(module.c_str()); #endif - if (functions == NULL) throw PDNSException("Cannot find PKCS#11 module " + module); - functions->C_Initialize(NULL); // initialize the module in case it hasn't been done yet. + if (functions == nullptr) throw PDNSException("Cannot find PKCS#11 module " + module); + functions->C_Initialize(nullptr); // initialize the module in case it hasn't been done yet. // try to locate a slot _CK_SLOT_INFO info; @@ -743,7 +726,7 @@ std::shared_ptr Pkcs11Slot::GetSlot(const std::string& module, const } // store slot - pkcs11_slots[sidx] = std::make_shared(functions, slotId); + pkcs11_slots[sidx] = std::make_shared>(Pkcs11Slot(functions, slotId)); return pkcs11_slots[sidx]; } @@ -758,12 +741,12 @@ std::shared_ptr Pkcs11Token::GetToken(const std::string& module, co std::map >::iterator tokenIter; if ((tokenIter = pkcs11_tokens.find(tidx)) != pkcs11_tokens.end()) return tokenIter->second; - std::shared_ptr slot = Pkcs11Slot::GetSlot(module, tokenId); + std::shared_ptr> slot = Pkcs11Slot::GetSlot(module, tokenId); pkcs11_tokens[tidx] = std::make_shared(slot, label, pub_label); return pkcs11_tokens[tidx]; } -Pkcs11Token::Pkcs11Token(const std::shared_ptr& slot, const std::string& label, const std::string& pub_label) : +Pkcs11Token::Pkcs11Token(const std::shared_ptr>& slot, const std::string& label, const std::string& pub_label) : d_slot(slot), d_bits(0), d_label(label), @@ -772,7 +755,7 @@ Pkcs11Token::Pkcs11Token(const std::shared_ptr& slot, const std::str d_err(0) { // open a session - if (this->d_slot->LoggedIn()) LoadAttributes(); + if (this->d_slot->lock()->LoggedIn()) LoadAttributes(); } Pkcs11Token::~Pkcs11Token() { @@ -780,10 +763,9 @@ Pkcs11Token::~Pkcs11Token() { bool PKCS11ModuleSlotLogin(const std::string& module, const string& tokenId, const std::string& pin) { - std::shared_ptr slot; - slot = Pkcs11Slot::GetSlot(module, tokenId); - if (slot->LoggedIn()) return true; // no point failing - return slot->Login(pin); + std::shared_ptr> slot = Pkcs11Slot::GetSlot(module, tokenId); + if (slot->lock()->LoggedIn()) return true; // no point failing + return slot->lock()->Login(pin); } PKCS11DNSCryptoKeyEngine::PKCS11DNSCryptoKeyEngine(unsigned int algorithm): DNSCryptoKeyEngine(algorithm) {} @@ -809,7 +791,7 @@ void PKCS11DNSCryptoKeyEngine::create(unsigned int bits) { throw PDNSException("pkcs11: unsupported algorithm "+std::to_string(d_algorithm)+ " for key pair generation"); } - mech.pParameter = NULL; + mech.pParameter = nullptr; mech.ulParameterLen = 0; if (mech.mechanism == CKM_RSA_PKCS_KEY_PAIR_GEN) { @@ -877,7 +859,7 @@ std::string PKCS11DNSCryptoKeyEngine::sign(const std::string& msg) const { CK_MECHANISM mech; mech.mechanism = dnssec2smech[d_algorithm]; - mech.pParameter = NULL; + mech.pParameter = nullptr; mech.ulParameterLen = 0; if (mech.mechanism == CKM_ECDSA) { @@ -892,7 +874,7 @@ std::string PKCS11DNSCryptoKeyEngine::hash(const std::string& msg) const { std::string result; CK_MECHANISM mech; mech.mechanism = dnssec2hmech[d_algorithm]; - mech.pParameter = NULL; + mech.pParameter = nullptr; mech.ulParameterLen = 0; std::shared_ptr d_slot; d_slot = Pkcs11Token::GetToken(d_module, d_slot_id, d_label, d_pub_label); @@ -933,7 +915,7 @@ bool PKCS11DNSCryptoKeyEngine::verify(const std::string& msg, const std::string& CK_MECHANISM mech; mech.mechanism = dnssec2smech[d_algorithm]; - mech.pParameter = NULL; + mech.pParameter = nullptr; mech.ulParameterLen = 0; if (mech.mechanism == CKM_ECDSA) { return (d_slot->Verify(this->hash(msg), signature, &mech)==0); @@ -990,17 +972,14 @@ int PKCS11DNSCryptoKeyEngine::getBits() const { }; DNSCryptoKeyEngine::storvector_t PKCS11DNSCryptoKeyEngine::convertToISCVector() const { - storvector_t storvect; - typedef std::vector > outputs_t; - outputs_t outputs; - - boost::assign::push_back(storvect) - (make_pair("Algorithm", std::to_string(d_algorithm))) - (make_pair("Engine", d_module)) - (make_pair("Slot", d_slot_id)) - (make_pair("PIN", d_pin)) - (make_pair("Label", d_label)) - (make_pair("PubLabel", d_pub_label)); + auto storvect = storvector_t{ + {"Algorithm", std::to_string(d_algorithm)}, + {"Engine", d_module}, + {"Slot", d_slot_id}, + {"PIN", d_pin}, + {"Label", d_label}, + {"PubLabel", d_pub_label}, + }; return storvect; }; diff --git a/pdns/protozero.cc b/pdns/protozero.cc index 28985f393fc1..7d13a9cb8411 100644 --- a/pdns/protozero.cc +++ b/pdns/protozero.cc @@ -56,11 +56,11 @@ void pdns::ProtoZero::Message::encodeDNSName(protozero::pbf_writer& pbf, std::st // leaving the block will cause the sub writer to compute how much was written based on the new size and update the size accordingly } -void pdns::ProtoZero::Message::setRequest(const boost::uuids::uuid& uniqueId, const ComboAddress& requestor, const ComboAddress& local, const DNSName& qname, uint16_t qtype, uint16_t qclass, uint16_t id, bool tcp, size_t len) +void pdns::ProtoZero::Message::setRequest(const boost::uuids::uuid& uniqueId, const ComboAddress& requestor, const ComboAddress& local, const DNSName& qname, uint16_t qtype, uint16_t qclass, uint16_t id, pdns::ProtoZero::Message::TransportProtocol proto, size_t len) { setMessageIdentity(uniqueId); setSocketFamily(requestor.sin4.sin_family); - setSocketProtocol(tcp); + setSocketProtocol(proto); setFrom(requestor); setTo(local); setInBytes(len); diff --git a/pdns/protozero.hh b/pdns/protozero.hh index a7a65e238835..61d009ac951b 100644 --- a/pdns/protozero.hh +++ b/pdns/protozero.hh @@ -35,11 +35,13 @@ namespace pdns { enum class MetaValueField : protozero::pbf_tag_type { stringVal = 1, intVal = 2 }; enum class MetaField : protozero::pbf_tag_type { key = 1, value = 2 }; + enum class Event : protozero::pbf_tag_type { ts = 1, event = 2, start = 3, boolVal = 4, intVal = 5, stringVal = 6, bytesVal = 7, custom = 8 }; enum class MessageType : int32_t { DNSQueryType = 1, DNSResponseType = 2, DNSOutgoingQueryType = 3, DNSIncomingResponseType = 4 }; - enum class Field : protozero::pbf_tag_type { type = 1, messageId = 2, serverIdentity = 3, socketFamily = 4, socketProtocol = 5, from = 6, to = 7, inBytes = 8, timeSec = 9, timeUsec = 10, id = 11, question = 12, response = 13, originalRequestorSubnet = 14, requestorId = 15, initialRequestId = 16, deviceId = 17, newlyObservedDomain = 18, deviceName = 19, fromPort = 20, toPort = 21, meta = 22 }; + enum class Field : protozero::pbf_tag_type { type = 1, messageId = 2, serverIdentity = 3, socketFamily = 4, socketProtocol = 5, from = 6, to = 7, inBytes = 8, timeSec = 9, timeUsec = 10, id = 11, question = 12, response = 13, originalRequestorSubnet = 14, requestorId = 15, initialRequestId = 16, deviceId = 17, newlyObservedDomain = 18, deviceName = 19, fromPort = 20, toPort = 21, meta = 22, trace = 23 }; enum class QuestionField : protozero::pbf_tag_type { qName = 1, qType = 2, qClass = 3 }; enum class ResponseField : protozero::pbf_tag_type { rcode = 1, rrs = 2, appliedPolicy = 3, tags = 4, queryTimeSec = 5, queryTimeUsec = 6, appliedPolicyType = 7, appliedPolicyTrigger = 8, appliedPolicyHit = 9, appliedPolicyKind = 10, validationState = 11 }; enum class RRField : protozero::pbf_tag_type { name = 1, type = 2, class_ = 3, ttl = 4, rdata = 5, udr = 6 }; + enum class TransportProtocol : protozero::pbf_tag_type { UDP = 1, TCP = 2, DoT = 3, DoH = 4, DNSCryptUDP = 5, DNSCryptTCP = 6 }; Message(std::string& buffer): d_buffer(buffer), d_message{d_buffer} { @@ -50,7 +52,7 @@ namespace pdns { Message& operator=(const Message&) = delete; Message& operator=(Message&&) = delete; - void setRequest(const boost::uuids::uuid& uniqueId, const ComboAddress& requestor, const ComboAddress& local, const DNSName& qname, uint16_t qtype, uint16_t qclass, uint16_t id, bool tcp, size_t len); + void setRequest(const boost::uuids::uuid& uniqueId, const ComboAddress& requestor, const ComboAddress& local, const DNSName& qname, uint16_t qtype, uint16_t qclass, uint16_t id, TransportProtocol proto, size_t len); void setResponse(const DNSName& qname, uint16_t qtype, uint16_t qclass); void setType(MessageType mtype) @@ -73,9 +75,9 @@ namespace pdns { add_enum(d_message, Field::socketFamily, family == AF_INET ? 1 : 2); } - void setSocketProtocol(bool tcp) + void setSocketProtocol(TransportProtocol proto) { - add_enum(d_message, Field::socketProtocol, tcp ? 2 : 1); + add_enum(d_message, Field::socketProtocol, static_cast(proto)); } void setFrom(const ComboAddress& ca) diff --git a/pdns/proxy-protocol.cc b/pdns/proxy-protocol.cc index 2b07f63f963e..b7c7270e79ba 100644 --- a/pdns/proxy-protocol.cc +++ b/pdns/proxy-protocol.cc @@ -243,11 +243,11 @@ template ssize_t parseProxyHeader(const Container& header, b return 0; } - values.push_back({ std::string(reinterpret_cast(&header.at(pos)), len), type }); + values.push_back({std::string(reinterpret_cast(&header.at(pos)), len), type}); pos += len; } else { - values.push_back({ std::string(), type }); + values.push_back({"", type}); } remaining = got - pos; diff --git a/pdns/proxy-protocol.hh b/pdns/proxy-protocol.hh index b2beb16d7a90..373a750e944a 100644 --- a/pdns/proxy-protocol.hh +++ b/pdns/proxy-protocol.hh @@ -22,7 +22,7 @@ #pragma once -#include +#include "iputils.hh" struct ProxyProtocolValue { diff --git a/pdns/qtype.cc b/pdns/qtype.cc index b3d7d1408734..8f4302bcd540 100644 --- a/pdns/qtype.cc +++ b/pdns/qtype.cc @@ -101,7 +101,7 @@ static map swapElements(const map ret; for (const auto& n : names) { - ret.insert(make_pair(n.second, n.first)); + ret.emplace(n.second, n.first); } return ret; } diff --git a/pdns/rcpgenerator.cc b/pdns/rcpgenerator.cc index ddda75c9eb0d..43bc488ce9b9 100644 --- a/pdns/rcpgenerator.cc +++ b/pdns/rcpgenerator.cc @@ -347,8 +347,9 @@ void RecordTextReader::xfrSvcParamKeyVals(set& val) // We've reached a space or equals-sign or the end of the string (d_pos is at this char) string k = d_string.substr(pos, d_pos - pos); SvcParam::SvcParamKey key; + bool generic; try { - key = SvcParam::keyFromString(k); + key = SvcParam::keyFromString(k, generic); } catch (const std::invalid_argument &e) { throw RecordTextException(e.what()); } @@ -372,11 +373,22 @@ void RecordTextReader::xfrSvcParamKeyVals(set& val) break; case SvcParam::ipv4hint: /* fall-through */ case SvcParam::ipv6hint: { - vector value; - xfrSVCBValueList(value); vector hints; bool doAuto{false}; - try { + if (generic) { + string value; + xfrRFC1035CharString(value); + size_t len = key == SvcParam::ipv4hint ? 4 : 16; + if (value.size() % len != 0) { + throw RecordTextException(k + " in generic format has wrong number of bytes"); + } + for (size_t i=0; i(key), &value.at(i), len); + hints.push_back(hint); + } + } else { + vector value; + xfrSVCBValueList(value); for (auto const &v: value) { if (v == "auto") { doAuto = true; @@ -385,6 +397,8 @@ void RecordTextReader::xfrSvcParamKeyVals(set& val) } hints.push_back(ComboAddress(v)); } + } + try { auto p = SvcParam(key, std::move(hints)); p.setAutoHint(doAuto); val.insert(p); @@ -396,11 +410,41 @@ void RecordTextReader::xfrSvcParamKeyVals(set& val) } case SvcParam::alpn: { vector value; - xfrSVCBValueList(value); + if (generic) { + string v; + xfrRFC1035CharString(v); + size_t spos{0}, len; + while (spos < v.length()) { + len = v.at(spos); + spos += 1; + if (len > v.length() - spos) { + throw RecordTextException("Length of ALPN value goes over total length of alpn SVC Param"); + } + value.push_back(v.substr(spos, len)); + spos += len; + } + } else { + xfrSVCBValueList(value); + } val.insert(SvcParam(key, std::move(value))); break; } case SvcParam::mandatory: { + if (generic) { + string v; + xfrRFC1035CharString(v); + if (v.length() % 2 != 0) { + throw RecordTextException("Wrong number of bytes in SVC Param " + k); + } + std::set keys; + for (size_t i=0; i < v.length(); i += 2) { + uint16_t mand = (v.at(i) << 8); + mand += v.at(i+1); + keys.insert(SvcParam::SvcParamKey(mand)); + } + val.insert(SvcParam(key, std::move(keys))); + break; + } vector parts; xfrSVCBValueList(parts); set values(parts.begin(), parts.end()); @@ -409,22 +453,36 @@ void RecordTextReader::xfrSvcParamKeyVals(set& val) } case SvcParam::port: { uint16_t port; - xfr16BitInt(port); + if (generic) { + string v; + xfrRFC1035CharString(v); + if (v.length() != 2) { + throw RecordTextException("port in generic format has the wrong length, expected 2, got " + std::to_string(v.length())); + } + port = (v.at(0) << 8); + port += v.at(1); + } else { + xfr16BitInt(port); + } val.insert(SvcParam(key, port)); break; } case SvcParam::ech: { - bool haveQuote = d_string.at(d_pos) == '"'; - if (haveQuote) { - d_pos++; - } string value; - xfrBlobNoSpaces(value); - if (haveQuote) { - if (d_string.at(d_pos) != '"') { - throw RecordTextException("ech value starts, but does not end with a '\"' symbol"); + if (generic) { + xfrRFC1035CharString(value); + } else { + bool haveQuote = d_string.at(d_pos) == '"'; + if (haveQuote) { + d_pos++; + } + xfrBlobNoSpaces(value); + if (haveQuote) { + if (d_string.at(d_pos) != '"') { + throw RecordTextException("ech value starts, but does not end with a '\"' symbol"); + } + d_pos++; } - d_pos++; } val.insert(SvcParam(key, value)); break; diff --git a/pdns/rec-carbon.cc b/pdns/rec-carbon.cc index 55604fdbed0f..723e447296f0 100644 --- a/pdns/rec-carbon.cc +++ b/pdns/rec-carbon.cc @@ -13,75 +13,74 @@ GlobalStateHolder g_carbonConfig; void doCarbonDump(void*) { -try -{ - static thread_local auto configHolder = g_carbonConfig.getLocal(); + try { + static thread_local auto configHolder = g_carbonConfig.getLocal(); - auto config = *configHolder; - if (config.servers.empty()) { - return; - } + auto config = *configHolder; + if (config.servers.empty()) { + return; + } - if (config.namespace_name.empty()) { - config.namespace_name = "pdns"; - } + if (config.namespace_name.empty()) { + config.namespace_name = "pdns"; + } - if (config.hostname.empty()) { - try { - config.hostname = getCarbonHostName(); + if (config.hostname.empty()) { + try { + config.hostname = getCarbonHostName(); + } + catch (const std::exception& e) { + throw std::runtime_error(std::string("The 'carbon-ourname' setting has not been set and we are unable to determine the system's hostname: ") + e.what()); + } } - catch(const std::exception& e) { - throw std::runtime_error(std::string("The 'carbon-ourname' setting has not been set and we are unable to determine the system's hostname: ") + e.what()); + if (config.instance_name.empty()) { + config.instance_name = "recursor"; } - } - if (config.instance_name.empty()) { - config.instance_name = "recursor"; - } - registerAllStats(); - PacketBuffer msg; - for (const auto& carbonServer: config.servers) { - ComboAddress remote(carbonServer, 2003); - Socket s(remote.sin4.sin_family, SOCK_STREAM); - s.setNonBlocking(); - std::shared_ptr tlsCtx{nullptr}; - const struct timeval timeout{g_networkTimeoutMsec / 1000, static_cast(g_networkTimeoutMsec) % 1000 * 1000}; - auto handler = std::make_shared("", s.releaseHandle(), timeout, tlsCtx, time(nullptr)); - handler->tryConnect(SyncRes::s_tcp_fast_open_connect, remote);// we do the connect so the first attempt happens while we gather stats + registerAllStats(); + PacketBuffer msg; + for (const auto& carbonServer : config.servers) { + ComboAddress remote(carbonServer, 2003); + Socket s(remote.sin4.sin_family, SOCK_STREAM); + s.setNonBlocking(); + std::shared_ptr tlsCtx{nullptr}; + const struct timeval timeout + { + g_networkTimeoutMsec / 1000, static_cast(g_networkTimeoutMsec) % 1000 * 1000 + }; + auto handler = std::make_shared("", s.releaseHandle(), timeout, tlsCtx, time(nullptr)); + handler->tryConnect(SyncRes::s_tcp_fast_open_connect, remote); // we do the connect so the first attempt happens while we gather stats - if(msg.empty()) { - auto all = getAllStatsMap(StatComponent::Carbon); + if (msg.empty()) { + auto all = getAllStatsMap(StatComponent::Carbon); - ostringstream str; - time_t now=time(0); + ostringstream str; + time_t now = time(0); - for(const auto& val : all) { - str<close(); } - handler->close(); } - } -catch (const PDNSException& e) -{ - g_log< g_luaconfs; +GlobalStateHolder g_luaconfs; /* SO HOW DOES THIS WORK! AND PLEASE PAY ATTENTION! This function can be called at any time. It is expected to overwrite all the contents @@ -34,8 +34,8 @@ GlobalStateHolder g_luaconfs; LuaConfigItems::LuaConfigItems() { DNSName root("."); // don't use g_rootdnsname here, it might not exist yet - for (const auto &dsRecord : rootDSs) { - auto ds=std::dynamic_pointer_cast(DSRecordContent::make(dsRecord)); + for (const auto& dsRecord : rootDSs) { + auto ds = std::dynamic_pointer_cast(DSRecordContent::make(dsRecord)); dsAnchors[root].insert(*ds); } } @@ -46,12 +46,12 @@ template typename C::value_type::second_type constGet(const C& c, const std::string& name) { auto iter = c.find(name); - if(iter == c.end()) + if (iter == c.end()) return 0; return iter->second; } -typedef std::unordered_map> > > rpzOptions_t; +typedef std::unordered_map>>> rpzOptions_t; static void parseRPZParameters(rpzOptions_t& have, std::shared_ptr& zone, std::string& polName, boost::optional& defpol, bool& defpolOverrideLocal, uint32_t& maxTTL) { @@ -59,14 +59,14 @@ static void parseRPZParameters(rpzOptions_t& have, std::shared_ptr(have["policyName"]); } if (have.count("defpol")) { - defpol=DNSFilterEngine::Policy(); + defpol = DNSFilterEngine::Policy(); defpol->d_kind = (DNSFilterEngine::PolicyKind)boost::get(have["defpol"]); defpol->setName(polName); - if(defpol->d_kind == DNSFilterEngine::PolicyKind::Custom) { + if (defpol->d_kind == DNSFilterEngine::PolicyKind::Custom) { defpol->d_custom.push_back(DNSRecordContent::mastermake(QType::CNAME, QClass::IN, boost::get(have["defcontent"]))); - if(have.count("defttl")) + if (have.count("defttl")) defpol->d_ttl = static_cast(boost::get(have["defttl"])); else defpol->d_ttl = -1; // get it from the zone @@ -109,7 +109,7 @@ static void parseRPZParameters(rpzOptions_t& have, std::shared_ptr > > > protobufOptions_t; +typedef std::unordered_map>>> protobufOptions_t; static void parseProtobufOptions(boost::optional vars, ProtobufExportConfig& config) { @@ -148,7 +148,7 @@ static void parseProtobufOptions(boost::optional vars, Protob if (vars->count("exportTypes")) { config.exportTypes.clear(); - auto types = boost::get>>((*vars)["exportTypes"]); + auto types = boost::get>>((*vars)["exportTypes"]); for (const auto& pair : types) { const auto type = pair.second; bool found = false; @@ -169,7 +169,7 @@ static void parseProtobufOptions(boost::optional vars, Protob } #ifdef HAVE_FSTRM -typedef std::unordered_map > > > frameStreamOptions_t; +typedef std::unordered_map>>> frameStreamOptions_t; static void parseFrameStreamOptions(boost::optional vars, FrameStreamExportConfig& config) { @@ -211,7 +211,7 @@ static void rpzPrimary(LuaConfigItems& lci, luaConfigDelayedThreads& delayedThre bool defpolOverrideLocal = true; std::shared_ptr zone = std::make_shared(); TSIGTriplet tt; - uint32_t refresh=0; + uint32_t refresh = 0; size_t maxReceivedXFRMBytes = 0; uint16_t axfrTimeout = 20; uint32_t maxTTL = std::numeric_limits::max(); @@ -238,37 +238,37 @@ static void rpzPrimary(LuaConfigItems& lci, luaConfigDelayedThreads& delayedThre auto& have = *options; parseRPZParameters(have, zone, polName, defpol, defpolOverrideLocal, maxTTL); - if(have.count("tsigname")) { - tt.name=DNSName(toLower(boost::get(have["tsigname"]))); - tt.algo=DNSName(toLower(boost::get(have[ "tsigalgo"]))); - if(B64Decode(boost::get(have[ "tsigsecret"]), tt.secret)) + if (have.count("tsigname")) { + tt.name = DNSName(toLower(boost::get(have["tsigname"]))); + tt.algo = DNSName(toLower(boost::get(have["tsigalgo"]))); + if (B64Decode(boost::get(have["tsigsecret"]), tt.secret)) throw std::runtime_error("TSIG secret is not valid Base-64 encoded"); } - if(have.count("refresh")) { + if (have.count("refresh")) { refresh = boost::get(have["refresh"]); if (refresh == 0) { - g_log<(boost::get(have["maxReceivedMBytes"])); } - if(have.count("localAddress")) { + if (have.count("localAddress")) { localAddress = ComboAddress(boost::get(have["localAddress"])); } - if(have.count("axfrTimeout")) { + if (have.count("axfrTimeout")) { axfrTimeout = static_cast(boost::get(have["axfrTimeout"])); } - if(have.count("seedFile")) { + if (have.count("seedFile")) { seedFile = boost::get(have["seedFile"]); } - if(have.count("dumpFile")) { + if (have.count("dumpFile")) { dumpFile = boost::get(have["dumpFile"]); } } @@ -277,7 +277,7 @@ static void rpzPrimary(LuaConfigItems& lci, luaConfigDelayedThreads& delayedThre // We were passed a localAddress, check if its AF matches the primaries' for (const auto& primary : primaries) { if (localAddress.sin4.sin_family != primary.sin4.sin_family) { - throw PDNSException("Primary address("+primary.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+")."); + throw PDNSException("Primary address(" + primary.toString() + ") is not of the same Address Family as the local address (" + localAddress.toString() + ")."); } } } @@ -288,7 +288,7 @@ static void rpzPrimary(LuaConfigItems& lci, luaConfigDelayedThreads& delayedThre zoneIdx = lci.dfe.addZone(zone); if (!seedFile.empty()) { - g_log<getDomain().toString() + ") has no SOA record"); } } - catch(const PDNSException& e) { - g_log<clear(); } - catch(const std::exception& e) { - g_log<clear(); } } } - catch(const std::exception& e) { - g_log<generation + 1; // pdnslog here is compatible with pdnslog in lua-base4.cc. - Lua.writeFunction("pdnslog", [](const std::string& msg, boost::optional loglevel) { g_log << (Logger::Urgency)loglevel.get_value_or(Logger::Warning) << msg< loglevel) { g_log << (Logger::Urgency)loglevel.get_value_or(Logger::Warning) << msg << endl; }); std::unordered_map> pdns_table; pdns_table["loglevels"] = std::unordered_map{ {"Alert", LOG_ALERT}, @@ -347,295 +347,365 @@ void loadRecursorLuaConfig(const std::string& fname, luaConfigDelayedThreads& de {"Info", LOG_INFO}, {"Notice", LOG_NOTICE}, {"Warning", LOG_WARNING}, - {"Error", LOG_ERR} - }; + {"Error", LOG_ERR}}; Lua.writeVariable("pdns", pdns_table); Lua.writeFunction("clearSortlist", [&lci]() { lci.sortlist.clear(); }); - + /* we can get: "1.2.3.4" {"1.2.3.4", "4.5.6.7"} {"1.2.3.4", {"4.5.6.7", "8.9.10.11"}} */ - map pmap{ - {"NoAction", DNSFilterEngine::PolicyKind::NoAction}, + map pmap{ + {"NoAction", DNSFilterEngine::PolicyKind::NoAction}, {"Drop", DNSFilterEngine::PolicyKind::Drop}, {"NXDOMAIN", DNSFilterEngine::PolicyKind::NXDOMAIN}, {"NODATA", DNSFilterEngine::PolicyKind::NODATA}, {"Truncate", DNSFilterEngine::PolicyKind::Truncate}, - {"Custom", DNSFilterEngine::PolicyKind::Custom} - }; + {"Custom", DNSFilterEngine::PolicyKind::Custom}}; Lua.writeVariable("Policy", pmap); Lua.writeFunction("rpzFile", [&lci](const string& filename, boost::optional options) { - try { - boost::optional defpol; - bool defpolOverrideLocal = true; - std::string polName("rpzFile"); - std::shared_ptr zone = std::make_shared(); - uint32_t maxTTL = std::numeric_limits::max(); - if (options) { - auto& have = *options; - parseRPZParameters(have, zone, polName, defpol, defpolOverrideLocal, maxTTL); - } - g_log<setName(polName); - loadRPZFromFile(filename, zone, defpol, defpolOverrideLocal, maxTTL); - lci.dfe.addZone(zone); - g_log< defpol; + bool defpolOverrideLocal = true; + std::string polName("rpzFile"); + std::shared_ptr zone = std::make_shared(); + uint32_t maxTTL = std::numeric_limits::max(); + if (options) { + auto& have = *options; + parseRPZParameters(have, zone, polName, defpol, defpolOverrideLocal, maxTTL); } - }); + g_log << Logger::Warning << "Loading RPZ from file '" << filename << "'" << endl; + zone->setName(polName); + loadRPZFromFile(filename, zone, defpol, defpolOverrideLocal, maxTTL); + lci.dfe.addZone(zone); + g_log << Logger::Warning << "Done loading RPZ from file '" << filename << "'" << endl; + } + catch (const std::exception& e) { + g_log << Logger::Error << "Unable to load RPZ zone from '" << filename << "': " << e.what() << endl; + } + }); - Lua.writeFunction("rpzMaster", [&lci, &delayedThreads](const boost::variant > >& primaries_, const string& zoneName, boost::optional options) { - g_log<>>& primaries_, const string& zoneName, boost::optional options) { + g_log << Logger::Warning << "'rpzMaster' is deprecated and will be removed in a future release, use 'rpzPrimary' instead" << endl; rpzPrimary(lci, delayedThreads, primaries_, zoneName, options); - }); - Lua.writeFunction("rpzPrimary", [&lci, &delayedThreads](const boost::variant > >& primaries_, const string& zoneName, boost::optional options) { + }); + Lua.writeFunction("rpzPrimary", [&lci, &delayedThreads](const boost::variant>>& primaries_, const string& zoneName, boost::optional options) { rpzPrimary(lci, delayedThreads, primaries_, zoneName, options); - }); - - typedef vector > > > > argvec_t; - Lua.writeFunction("addSortList", - [&lci](const std::string& formask_, - const boost::variant& masks, - boost::optional order_) - { - try { - Netmask formask(formask_); - int order = order_ ? (*order_) : lci.sortlist.getMaxOrder(formask)+1; - if(auto str = boost::get(&masks)) - lci.sortlist.addEntry(formask, Netmask(*str), order); - else { - - auto vec = boost::get(&masks); - for(const auto& e : *vec) { - if(auto s = boost::get(&e.second)) { - lci.sortlist.addEntry(formask, Netmask(*s), order); - } - else { - const auto& v =boost::get > >(e.second); - for(const auto& entry : v) - lci.sortlist.addEntry(formask, Netmask(entry.second), order); - } - ++order; - } - } - } - catch(std::exception& e) { - g_log<> zoneToCacheOptions_t; + + Lua.writeFunction("zoneToCache", [&delayedThreads](const string& zoneName, const string& method, const boost::variant>>& srcs, boost::optional options) { + try { + RecZoneToCache::Config conf; + DNSName validZoneName(zoneName); + conf.d_zone = zoneName; + const set methods = {"axfr", "url", "file"}; + if (methods.count(method) == 0) { + throw std::runtime_error("unknwon method '" + method + "'"); + } + conf.d_method = method; + if (srcs.type() == typeid(std::string)) { + conf.d_sources.push_back(boost::get(srcs)); + } + else { + for (const auto& src : boost::get>>(srcs)) { + conf.d_sources.push_back(src.second); + } + } + if (conf.d_sources.size() == 0) { + throw std::runtime_error("at least one source required"); + } + if (options) { + auto& have = *options; + if (have.count("timeout")) { + conf.d_timeout = boost::get(have.at("timeout")); + } + if (have.count("tsigname")) { + conf.d_tt.name = DNSName(toLower(boost::get(have.at("tsigname")))); + conf.d_tt.algo = DNSName(toLower(boost::get(have.at("tsigalgo")))); + if (B64Decode(boost::get(have.at("tsigsecret")), conf.d_tt.secret)) { + throw std::runtime_error("TSIG secret is not valid Base-64 encoded"); + } + } + if (have.count("maxReceivedMBytes")) { + conf.d_maxReceivedBytes = static_cast(boost::get(have.at("maxReceivedMBytes"))); + conf.d_maxReceivedBytes *= 1024 * 1024; + } + if (have.count("localAddress")) { + conf.d_local = ComboAddress(boost::get(have.at("localAddress"))); + } + if (have.count("refreshPeriod")) { + conf.d_refreshPeriod = boost::get(have.at("refreshPeriod")); + } + if (have.count("retryOnErrorPeriod")) { + conf.d_retryOnError = boost::get(have.at("retryOnErrorPeriod")); + } + } + + delayedThreads.ztcConfigs.push_back(conf); + } + catch (const std::exception& e) { + g_log << Logger::Error << "Problem configuring zoneToCache for zone '" << zoneName << "': " << e.what() << endl; + } + }); + + typedef vector>>>> argvec_t; + Lua.writeFunction("addSortList", + [&lci](const std::string& formask_, + const boost::variant& masks, + boost::optional order_) { + try { + Netmask formask(formask_); + int order = order_ ? (*order_) : lci.sortlist.getMaxOrder(formask) + 1; + if (auto str = boost::get(&masks)) + lci.sortlist.addEntry(formask, Netmask(*str), order); + else { + + auto vec = boost::get(&masks); + for (const auto& e : *vec) { + if (auto s = boost::get(&e.second)) { + lci.sortlist.addEntry(formask, Netmask(*s), order); + } + else { + const auto& v = boost::get>>(e.second); + for (const auto& entry : v) + lci.sortlist.addEntry(formask, Netmask(entry.second), order); + } + ++order; + } + } + } + catch (std::exception& e) { + g_log << Logger::Error << "Error in addSortList: " << e.what() << endl; + } + }); Lua.writeFunction("addTA", [&lci](const std::string& who, const std::string& what) { - warnIfDNSSECDisabled("Warning: adding Trust Anchor for DNSSEC (addTA), but dnssec is set to 'off'!"); - DNSName zone(who); - auto ds = std::dynamic_pointer_cast(DSRecordContent::make(what)); - lci.dsAnchors[zone].insert(*ds); + warnIfDNSSECDisabled("Warning: adding Trust Anchor for DNSSEC (addTA), but dnssec is set to 'off'!"); + DNSName zone(who); + auto ds = std::dynamic_pointer_cast(DSRecordContent::make(what)); + lci.dsAnchors[zone].insert(*ds); }); Lua.writeFunction("clearTA", [&lci](boost::optional who) { - warnIfDNSSECDisabled("Warning: removing Trust Anchor for DNSSEC (clearTA), but dnssec is set to 'off'!"); - if(who) - lci.dsAnchors.erase(DNSName(*who)); - else - lci.dsAnchors.clear(); - }); + warnIfDNSSECDisabled("Warning: removing Trust Anchor for DNSSEC (clearTA), but dnssec is set to 'off'!"); + if (who) + lci.dsAnchors.erase(DNSName(*who)); + else + lci.dsAnchors.clear(); + }); /* Remove in 4.3 */ Lua.writeFunction("addDS", [&lci](const std::string& who, const std::string& what) { - warnIfDNSSECDisabled("Warning: adding Trust Anchor for DNSSEC (addDS), but dnssec is set to 'off'!"); - g_log<(DSRecordContent::make(what)); - lci.dsAnchors[zone].insert(*ds); + warnIfDNSSECDisabled("Warning: adding Trust Anchor for DNSSEC (addDS), but dnssec is set to 'off'!"); + g_log << Logger::Warning << "addDS is deprecated and will be removed in the future, switch to addTA" << endl; + DNSName zone(who); + auto ds = std::dynamic_pointer_cast(DSRecordContent::make(what)); + lci.dsAnchors[zone].insert(*ds); }); /* Remove in 4.3 */ Lua.writeFunction("clearDS", [&lci](boost::optional who) { - g_log< why) { - warnIfDNSSECDisabled("Warning: adding Negative Trust Anchor for DNSSEC (addNTA), but dnssec is set to 'off'!"); - if(why) - lci.negAnchors[DNSName(who)] = static_cast(*why); - else - lci.negAnchors[DNSName(who)] = ""; - }); + warnIfDNSSECDisabled("Warning: adding Negative Trust Anchor for DNSSEC (addNTA), but dnssec is set to 'off'!"); + if (why) + lci.negAnchors[DNSName(who)] = static_cast(*why); + else + lci.negAnchors[DNSName(who)] = ""; + }); Lua.writeFunction("clearNTA", [&lci](boost::optional who) { - warnIfDNSSECDisabled("Warning: removing Negative Trust Anchor for DNSSEC (clearNTA), but dnssec is set to 'off'!"); - if(who) - lci.negAnchors.erase(DNSName(*who)); - else - lci.negAnchors.clear(); - }); + warnIfDNSSECDisabled("Warning: removing Negative Trust Anchor for DNSSEC (clearNTA), but dnssec is set to 'off'!"); + if (who) + lci.negAnchors.erase(DNSName(*who)); + else + lci.negAnchors.clear(); + }); Lua.writeFunction("readTrustAnchorsFromFile", [&lci](const std::string& fnamearg, const boost::optional interval) { - uint32_t realInterval = 24; - if (interval) { - realInterval = static_cast(*interval); - } - warnIfDNSSECDisabled("Warning: reading Trust Anchors from file (readTrustAnchorsFromFile), but dnssec is set to 'off'!"); - lci.trustAnchorFileInfo.fname = fnamearg; - lci.trustAnchorFileInfo.interval = realInterval; - updateTrustAnchorsFromFile(fnamearg, lci.dsAnchors); - }); + uint32_t realInterval = 24; + if (interval) { + realInterval = static_cast(*interval); + } + warnIfDNSSECDisabled("Warning: reading Trust Anchors from file (readTrustAnchorsFromFile), but dnssec is set to 'off'!"); + lci.trustAnchorFileInfo.fname = fnamearg; + lci.trustAnchorFileInfo.interval = realInterval; + updateTrustAnchorsFromFile(fnamearg, lci.dsAnchors); + }); Lua.writeFunction("setProtobufMasks", [&lci](const uint8_t maskV4, uint8_t maskV6) { - lci.protobufMaskV4 = maskV4; - lci.protobufMaskV6 = maskV6; - }); + lci.protobufMaskV4 = maskV4; + lci.protobufMaskV6 = maskV6; + }); Lua.writeFunction("protobufServer", [&lci](boost::variant> servers, boost::optional vars) { - if (!lci.protobufExportConfig.enabled) { - - lci.protobufExportConfig.enabled = true; + if (!lci.protobufExportConfig.enabled) { - try { - if (servers.type() == typeid(std::string)) { - auto server = boost::get(servers); + lci.protobufExportConfig.enabled = true; - lci.protobufExportConfig.servers.emplace_back(server); - } - else { - auto serversMap = boost::get>(servers); - for (const auto& serverPair : serversMap) { - lci.protobufExportConfig.servers.emplace_back(serverPair.second); - } - } + try { + if (servers.type() == typeid(std::string)) { + auto server = boost::get(servers); - parseProtobufOptions(vars, lci.protobufExportConfig); - } - catch(std::exception& e) { - g_log<>(servers); + for (const auto& serverPair : serversMap) { + lci.protobufExportConfig.servers.emplace_back(serverPair.second); + } } - }); - Lua.writeFunction("outgoingProtobufServer", [&lci](boost::variant> servers, boost::optional vars) { - if (!lci.outgoingProtobufExportConfig.enabled) { + parseProtobufOptions(vars, lci.protobufExportConfig); + } + catch (std::exception& e) { + g_log << Logger::Error << "Error while adding protobuf logger: " << e.what() << endl; + } + catch (PDNSException& e) { + g_log << Logger::Error << "Error while adding protobuf logger: " << e.reason << endl; + } + } + else { + g_log << Logger::Error << "Only one protobufServer() directive can be configured, we already have " << lci.protobufExportConfig.servers.at(0).toString() << endl; + } + }); - lci.outgoingProtobufExportConfig.enabled = true; + Lua.writeFunction("outgoingProtobufServer", [&lci](boost::variant> servers, boost::optional vars) { + if (!lci.outgoingProtobufExportConfig.enabled) { - try { - if (servers.type() == typeid(std::string)) { - auto server = boost::get(servers); + lci.outgoingProtobufExportConfig.enabled = true; - lci.outgoingProtobufExportConfig.servers.emplace_back(server); - } - else { - auto serversMap = boost::get>(servers); - for (const auto& serverPair : serversMap) { - lci.outgoingProtobufExportConfig.servers.emplace_back(serverPair.second); - } - } + try { + if (servers.type() == typeid(std::string)) { + auto server = boost::get(servers); - parseProtobufOptions(vars, lci.outgoingProtobufExportConfig); - } - catch(std::exception& e) { - g_log<>(servers); + for (const auto& serverPair : serversMap) { + lci.outgoingProtobufExportConfig.servers.emplace_back(serverPair.second); } + } + + parseProtobufOptions(vars, lci.outgoingProtobufExportConfig); } - else { - g_log<> servers, boost::optional vars) { - if (!lci.frameStreamExportConfig.enabled) { - - lci.frameStreamExportConfig.enabled = true; - - try { - if (servers.type() == typeid(std::string)) { - auto server = boost::get(servers); - if (!boost::starts_with(server, "/")) { - ComboAddress parsecheck(server); - } - lci.frameStreamExportConfig.servers.emplace_back(server); - } - else { - auto serversMap = boost::get>(servers); - for (const auto& serverPair : serversMap) { - lci.frameStreamExportConfig.servers.emplace_back(serverPair.second); - } - } - - parseFrameStreamOptions(vars, lci.frameStreamExportConfig); - } - catch(std::exception& e) { - g_log<(servers); + if (!boost::starts_with(server, "/")) { + ComboAddress parsecheck(server); } - catch(PDNSException& e) { - g_log<>(servers); + for (const auto& serverPair : serversMap) { + lci.frameStreamExportConfig.servers.emplace_back(serverPair.second); } + } + + parseFrameStreamOptions(vars, lci.frameStreamExportConfig); } - else { - g_log<(rpzPrimary), std::get<1>(rpzPrimary), std::get<2>(rpzPrimary), std::get<3>(rpzPrimary), std::get<4>(rpzPrimary), std::get<5>(rpzPrimary), std::get<6>(rpzPrimary) * 1024 * 1024, std::get<7>(rpzPrimary), std::get<8>(rpzPrimary), std::get<9>(rpzPrimary), std::get<10>(rpzPrimary), std::get<11>(rpzPrimary), generation); t.detach(); } - catch(const std::exception& e) { - g_log< exportTypes = { QType::A, QType::AAAA, QType::CNAME }; + std::set exportTypes = {QType::A, QType::AAAA, QType::CNAME}; std::vector servers; uint64_t maxQueuedEntries{100}; uint16_t timeout{2}; @@ -55,24 +56,24 @@ struct FrameStreamExportConfig unsigned reopenInterval{0}; }; -struct TrustAnchorFileInfo { +struct TrustAnchorFileInfo +{ uint32_t interval{24}; std::string fname; }; -class LuaConfigItems +class LuaConfigItems { public: LuaConfigItems(); SortList sortlist; DNSFilterEngine dfe; TrustAnchorFileInfo trustAnchorFileInfo; // Used to update the Trust Anchors from file periodically - map dsAnchors; - map negAnchors; + map dsAnchors; + map negAnchors; ProtobufExportConfig protobufExportConfig; ProtobufExportConfig outgoingProtobufExportConfig; FrameStreamExportConfig frameStreamExportConfig; - /* we need to increment this every time the configuration is reloaded, so we know if we need to reload the protobuf remote loggers */ @@ -85,9 +86,10 @@ extern GlobalStateHolder g_luaconfs; struct luaConfigDelayedThreads { - std::vector, boost::optional, bool, uint32_t, size_t, TSIGTriplet, size_t, ComboAddress, uint16_t, uint32_t, std::shared_ptr, std::string> > rpzPrimaryThreads; + // Please make sure that the tuple below only contains value types since they are used as parameters in a thread ct + std::vector, boost::optional, bool, uint32_t, size_t, TSIGTriplet, size_t, ComboAddress, uint16_t, uint32_t, std::shared_ptr, std::string>> rpzPrimaryThreads; + std::vector ztcConfigs; }; void loadRecursorLuaConfig(const std::string& fname, luaConfigDelayedThreads& delayedThreads); void startLuaConfigDelayedThreads(const luaConfigDelayedThreads& delayedThreads, uint64_t generation); - diff --git a/pdns/rec-snmp.cc b/pdns/rec-snmp.cc index a63dd78c9673..d6867d86a472 100644 --- a/pdns/rec-snmp.cc +++ b/pdns/rec-snmp.cc @@ -13,128 +13,138 @@ #define RECURSOR_TRAPS_OID RECURSOR_OID, 10, 0 #define RECURSOR_TRAP_OBJECTS_OID RECURSOR_OID, 11 -static const oid trapReasonOID[] = { RECURSOR_TRAP_OBJECTS_OID, 1, 0 }; -static const oid customTrapOID[] = { RECURSOR_TRAPS_OID, 1 }; +static const oid trapReasonOID[] = {RECURSOR_TRAP_OBJECTS_OID, 1, 0}; +static const oid customTrapOID[] = {RECURSOR_TRAPS_OID, 1}; -static const oid questionsOID[] = { RECURSOR_STATS_OID, 1 }; -static const oid ipv6QuestionsOID[] = { RECURSOR_STATS_OID, 2 }; -static const oid tcpQuestionsOID[] = { RECURSOR_STATS_OID, 3 }; -static const oid cacheHitsOID[] = { RECURSOR_STATS_OID, 4 }; -static const oid cacheMissesOID[] = { RECURSOR_STATS_OID, 5 }; -static const oid cacheEntriesOID[] = { RECURSOR_STATS_OID, 6 }; -static const oid cacheBytesOID[] = { RECURSOR_STATS_OID, 7 }; -static const oid packetcacheHitsOID[] = { RECURSOR_STATS_OID, 8 }; -static const oid packetcacheMissesOID[] = { RECURSOR_STATS_OID, 9 }; -static const oid packetcacheEntriesOID[] = { RECURSOR_STATS_OID, 10 }; -static const oid packetcacheBytesOID[] = { RECURSOR_STATS_OID, 11 }; -static const oid mallocBytesOID[] = { RECURSOR_STATS_OID, 12 }; -static const oid servfailAnswersOID[] = { RECURSOR_STATS_OID, 13 }; -static const oid nxdomainAnswersOID[] = { RECURSOR_STATS_OID, 14 }; -static const oid noerrorAnswersOID[] = { RECURSOR_STATS_OID, 15 }; -static const oid unauthorizedUdpOID[] = { RECURSOR_STATS_OID, 16 }; -static const oid unauthorizedTcpOID[] = { RECURSOR_STATS_OID, 17 }; -static const oid tcpClientOverflowOID[] = { RECURSOR_STATS_OID, 18 }; -static const oid clientParseErrorsOID[] = { RECURSOR_STATS_OID, 19 }; -static const oid serverParseErrorsOID[] = { RECURSOR_STATS_OID, 20 }; -static const oid tooOldDropsOID[] = { RECURSOR_STATS_OID, 21 }; -static const oid answers01OID[] = { RECURSOR_STATS_OID, 22 }; -static const oid answers110OID[] = { RECURSOR_STATS_OID, 23 }; -static const oid answers10100OID[] = { RECURSOR_STATS_OID, 24 }; -static const oid answers1001000OID[] = { RECURSOR_STATS_OID, 25 }; -static const oid answersSlowOID[] = { RECURSOR_STATS_OID, 26 }; -static const oid auth4Answers01OID[] = { RECURSOR_STATS_OID, 27 }; -static const oid auth4Answers110OID[] = { RECURSOR_STATS_OID, 28 }; -static const oid auth4Answers10100OID[] = { RECURSOR_STATS_OID, 29 }; -static const oid auth4Answers1001000OID[] = { RECURSOR_STATS_OID, 30 }; -static const oid auth4AnswersslowOID[] = { RECURSOR_STATS_OID, 31 }; -static const oid auth6Answers01OID[] = { RECURSOR_STATS_OID, 32 }; -static const oid auth6Answers110OID[] = { RECURSOR_STATS_OID, 33 }; -static const oid auth6Answers10100OID[] = { RECURSOR_STATS_OID, 34 }; -static const oid auth6Answers1001000OID[] = { RECURSOR_STATS_OID, 35 }; -static const oid auth6AnswersSlowOID[] = { RECURSOR_STATS_OID, 36 }; -static const oid qaLatencyOID[] = { RECURSOR_STATS_OID, 37 }; -static const oid unexpectedPacketsOID[] = { RECURSOR_STATS_OID, 38 }; -static const oid caseMismatchesOID[] = { RECURSOR_STATS_OID, 39 }; -static const oid spoofPreventsOID[] = { RECURSOR_STATS_OID, 40 }; -static const oid nssetInvalidationsOID[] = { RECURSOR_STATS_OID, 41 }; -static const oid resourceLimitsOID[] = { RECURSOR_STATS_OID, 42 }; -static const oid overCapacityDropsOID[] = { RECURSOR_STATS_OID, 43 }; -static const oid policyDropsOID[] = { RECURSOR_STATS_OID, 44 }; -static const oid noPacketErrorOID[] = { RECURSOR_STATS_OID, 45 }; -static const oid dlgOnlyDropsOID[] = { RECURSOR_STATS_OID, 46 }; -static const oid ignoredPacketsOID[] = { RECURSOR_STATS_OID, 47 }; -static const oid maxMthreadStackOID[] = { RECURSOR_STATS_OID, 48 }; -static const oid negcacheEntriesOID[] = { RECURSOR_STATS_OID, 49 }; -static const oid throttleEntriesOID[] = { RECURSOR_STATS_OID, 50 }; -static const oid nsspeedsEntriesOID[] = { RECURSOR_STATS_OID, 51 }; -static const oid failedHostEntriesOID[] = { RECURSOR_STATS_OID, 52 }; -static const oid concurrentQueriesOID[] = { RECURSOR_STATS_OID, 53 }; -static const oid securityStatusOID[] = { RECURSOR_STATS_OID, 54 }; -static const oid outgoingTimeoutsOID[] = { RECURSOR_STATS_OID, 55 }; -static const oid outgoing4TimeoutsOID[] = { RECURSOR_STATS_OID, 56 }; -static const oid outgoing6TimeoutsOID[] = { RECURSOR_STATS_OID, 57 }; -static const oid tcpOutqueriesOID[] = { RECURSOR_STATS_OID, 58 }; -static const oid allOutqueriesOID[] = { RECURSOR_STATS_OID, 59 }; -static const oid ipv6OutqueriesOID[] = { RECURSOR_STATS_OID, 60 }; -static const oid throttledOutqueriesOID[] = { RECURSOR_STATS_OID, 61 }; -static const oid dontOutqueriesOID[] = { RECURSOR_STATS_OID, 62 }; -static const oid unreachablesOID[] = { RECURSOR_STATS_OID, 63 }; -static const oid chainResendsOID[] = { RECURSOR_STATS_OID, 64 }; -static const oid tcpClientsOID[] = { RECURSOR_STATS_OID, 65 }; +static const oid questionsOID[] = {RECURSOR_STATS_OID, 1}; +static const oid ipv6QuestionsOID[] = {RECURSOR_STATS_OID, 2}; +static const oid tcpQuestionsOID[] = {RECURSOR_STATS_OID, 3}; +static const oid cacheHitsOID[] = {RECURSOR_STATS_OID, 4}; +static const oid cacheMissesOID[] = {RECURSOR_STATS_OID, 5}; +static const oid cacheEntriesOID[] = {RECURSOR_STATS_OID, 6}; +static const oid cacheBytesOID[] = {RECURSOR_STATS_OID, 7}; +static const oid packetcacheHitsOID[] = {RECURSOR_STATS_OID, 8}; +static const oid packetcacheMissesOID[] = {RECURSOR_STATS_OID, 9}; +static const oid packetcacheEntriesOID[] = {RECURSOR_STATS_OID, 10}; +static const oid packetcacheBytesOID[] = {RECURSOR_STATS_OID, 11}; +static const oid mallocBytesOID[] = {RECURSOR_STATS_OID, 12}; +static const oid servfailAnswersOID[] = {RECURSOR_STATS_OID, 13}; +static const oid nxdomainAnswersOID[] = {RECURSOR_STATS_OID, 14}; +static const oid noerrorAnswersOID[] = {RECURSOR_STATS_OID, 15}; +static const oid unauthorizedUdpOID[] = {RECURSOR_STATS_OID, 16}; +static const oid unauthorizedTcpOID[] = {RECURSOR_STATS_OID, 17}; +static const oid tcpClientOverflowOID[] = {RECURSOR_STATS_OID, 18}; +static const oid clientParseErrorsOID[] = {RECURSOR_STATS_OID, 19}; +static const oid serverParseErrorsOID[] = {RECURSOR_STATS_OID, 20}; +static const oid tooOldDropsOID[] = {RECURSOR_STATS_OID, 21}; +static const oid answers01OID[] = {RECURSOR_STATS_OID, 22}; +static const oid answers110OID[] = {RECURSOR_STATS_OID, 23}; +static const oid answers10100OID[] = {RECURSOR_STATS_OID, 24}; +static const oid answers1001000OID[] = {RECURSOR_STATS_OID, 25}; +static const oid answersSlowOID[] = {RECURSOR_STATS_OID, 26}; +static const oid auth4Answers01OID[] = {RECURSOR_STATS_OID, 27}; +static const oid auth4Answers110OID[] = {RECURSOR_STATS_OID, 28}; +static const oid auth4Answers10100OID[] = {RECURSOR_STATS_OID, 29}; +static const oid auth4Answers1001000OID[] = {RECURSOR_STATS_OID, 30}; +static const oid auth4AnswersslowOID[] = {RECURSOR_STATS_OID, 31}; +static const oid auth6Answers01OID[] = {RECURSOR_STATS_OID, 32}; +static const oid auth6Answers110OID[] = {RECURSOR_STATS_OID, 33}; +static const oid auth6Answers10100OID[] = {RECURSOR_STATS_OID, 34}; +static const oid auth6Answers1001000OID[] = {RECURSOR_STATS_OID, 35}; +static const oid auth6AnswersSlowOID[] = {RECURSOR_STATS_OID, 36}; +static const oid qaLatencyOID[] = {RECURSOR_STATS_OID, 37}; +static const oid unexpectedPacketsOID[] = {RECURSOR_STATS_OID, 38}; +static const oid caseMismatchesOID[] = {RECURSOR_STATS_OID, 39}; +static const oid spoofPreventsOID[] = {RECURSOR_STATS_OID, 40}; +static const oid nssetInvalidationsOID[] = {RECURSOR_STATS_OID, 41}; +static const oid resourceLimitsOID[] = {RECURSOR_STATS_OID, 42}; +static const oid overCapacityDropsOID[] = {RECURSOR_STATS_OID, 43}; +static const oid policyDropsOID[] = {RECURSOR_STATS_OID, 44}; +static const oid noPacketErrorOID[] = {RECURSOR_STATS_OID, 45}; +static const oid dlgOnlyDropsOID[] = {RECURSOR_STATS_OID, 46}; +static const oid ignoredPacketsOID[] = {RECURSOR_STATS_OID, 47}; +static const oid maxMthreadStackOID[] = {RECURSOR_STATS_OID, 48}; +static const oid negcacheEntriesOID[] = {RECURSOR_STATS_OID, 49}; +static const oid throttleEntriesOID[] = {RECURSOR_STATS_OID, 50}; +static const oid nsspeedsEntriesOID[] = {RECURSOR_STATS_OID, 51}; +static const oid failedHostEntriesOID[] = {RECURSOR_STATS_OID, 52}; +static const oid concurrentQueriesOID[] = {RECURSOR_STATS_OID, 53}; +static const oid securityStatusOID[] = {RECURSOR_STATS_OID, 54}; +static const oid outgoingTimeoutsOID[] = {RECURSOR_STATS_OID, 55}; +static const oid outgoing4TimeoutsOID[] = {RECURSOR_STATS_OID, 56}; +static const oid outgoing6TimeoutsOID[] = {RECURSOR_STATS_OID, 57}; +static const oid tcpOutqueriesOID[] = {RECURSOR_STATS_OID, 58}; +static const oid allOutqueriesOID[] = {RECURSOR_STATS_OID, 59}; +static const oid ipv6OutqueriesOID[] = {RECURSOR_STATS_OID, 60}; +static const oid throttledOutqueriesOID[] = {RECURSOR_STATS_OID, 61}; +static const oid dontOutqueriesOID[] = {RECURSOR_STATS_OID, 62}; +static const oid unreachablesOID[] = {RECURSOR_STATS_OID, 63}; +static const oid chainResendsOID[] = {RECURSOR_STATS_OID, 64}; +static const oid tcpClientsOID[] = {RECURSOR_STATS_OID, 65}; #ifdef __linux__ -static const oid udpRecvbufErrorsOID[] = { RECURSOR_STATS_OID, 66 }; -static const oid udpSndbufErrorsOID[] = { RECURSOR_STATS_OID, 67 }; -static const oid udpNoportErrorsOID[] = { RECURSOR_STATS_OID, 68 }; -static const oid udpinErrorsOID[] = { RECURSOR_STATS_OID, 69 }; +static const oid udpRecvbufErrorsOID[] = {RECURSOR_STATS_OID, 66}; +static const oid udpSndbufErrorsOID[] = {RECURSOR_STATS_OID, 67}; +static const oid udpNoportErrorsOID[] = {RECURSOR_STATS_OID, 68}; +static const oid udpinErrorsOID[] = {RECURSOR_STATS_OID, 69}; #endif /* __linux__ */ -static const oid ednsPingMatchesOID[] = { RECURSOR_STATS_OID, 70 }; -static const oid ednsPingMismatchesOID[] = { RECURSOR_STATS_OID, 71 }; -static const oid dnssecQueriesOID[] = { RECURSOR_STATS_OID, 72 }; -static const oid nopingOutqueriesOID[] = { RECURSOR_STATS_OID, 73 }; -static const oid noednsOutqueriesOID[] = { RECURSOR_STATS_OID, 74 }; -static const oid uptimeOID[] = { RECURSOR_STATS_OID, 75 }; -static const oid realMemoryUsageOID[] = { RECURSOR_STATS_OID, 76 }; -static const oid fdUsageOID[] = { RECURSOR_STATS_OID, 77 }; -static const oid userMsecOID[] = { RECURSOR_STATS_OID, 78 }; -static const oid sysMsecOID[] = { RECURSOR_STATS_OID, 79 }; -static const oid dnssecValidationsOID[] = { RECURSOR_STATS_OID, 80 }; -static const oid dnssecResultInsecureOID[] = { RECURSOR_STATS_OID, 81 }; -static const oid dnssecResultSecureOID[] = { RECURSOR_STATS_OID, 82 }; -static const oid dnssecResultBogusOID[] = { RECURSOR_STATS_OID, 83 }; -static const oid dnssecResultIndeterminateOID[] = { RECURSOR_STATS_OID, 84 }; -static const oid dnssecResultNtaOID[] = { RECURSOR_STATS_OID, 85 }; -static const oid policyResultNoactionOID[] = { RECURSOR_STATS_OID, 86 }; -static const oid policyResultDropOID[] = { RECURSOR_STATS_OID, 87 }; -static const oid policyResultNxdomainOID[] = { RECURSOR_STATS_OID, 88 }; -static const oid policyResultNodataOID[] = { RECURSOR_STATS_OID, 89 }; -static const oid policyResultTruncateOID[] = { RECURSOR_STATS_OID, 90 }; -static const oid policyResultCustomOID[] = { RECURSOR_STATS_OID, 91 }; -static const oid queryPipeFullDropsOID[] = { RECURSOR_STATS_OID, 92 }; -static const oid truncatedDropsOID[] = { RECURSOR_STATS_OID, 93 }; -static const oid emptyQueriesOID[] = { RECURSOR_STATS_OID, 94 }; -static const oid dnssecAuthenticDataQueriesOID[] = { RECURSOR_STATS_OID, 95 }; -static const oid dnssecCheckDisabledQueriesOID[] = { RECURSOR_STATS_OID, 96 }; -static const oid variableResponsesOID[] = { RECURSOR_STATS_OID, 97 }; -static const oid specialMemoryUsageOID[] = { RECURSOR_STATS_OID, 98 }; -static const oid rebalancedQueriesOID[] = { RECURSOR_STATS_OID, 99 }; -static const oid qnameMinFallbackSuccessOID[] = { RECURSOR_STATS_OID, 100 }; -static const oid proxyProtocolInvalidOID[] = { RECURSOR_STATS_OID, 101 }; -static const oid recordCacheContendedOID[] = { RECURSOR_STATS_OID, 102 }; -static const oid recordCacheAcquiredOID[] = { RECURSOR_STATS_OID, 103 }; -static const oid nodLookupsDroppedOversizeOID[] = { RECURSOR_STATS_OID, 104 }; -static const oid taskQueuePushedOID[] = { RECURSOR_STATS_OID, 105 }; -static const oid taskQueueExpiredOID[] = { RECURSOR_STATS_OID, 106 }; -static const oid taskQueueSizeOID[] = { RECURSOR_STATS_OID, 107 }; -static const oid aggressiveNSECCacheEntriesOID[] = { RECURSOR_STATS_OID, 108 }; -static const oid aggressiveNSECCacheNSECHitsOID[] = { RECURSOR_STATS_OID, 109 }; -static const oid aggressiveNSECCacheNSEC3HitsOID[] = { RECURSOR_STATS_OID, 110 }; -static const oid aggressiveNSECCacheNSECWCHitsOID[] = { RECURSOR_STATS_OID, 111 }; -static const oid aggressiveNSECCacheNSEC3WCHitsOID[] = { RECURSOR_STATS_OID, 112 }; -static const oid dotOutqueriesOID [] = { RECURSOR_STATS_OID, 113 }; -static const oid dns64PrefixAnswers [] = { RECURSOR_STATS_OID, 114 }; -static const oid almostExpiredPushed [] = { RECURSOR_STATS_OID, 115 }; -static const oid almostExpiredRun [] = { RECURSOR_STATS_OID, 116 }; -static const oid almostExpiredExceptions [] = { RECURSOR_STATS_OID, 117 }; +static const oid ednsPingMatchesOID[] = {RECURSOR_STATS_OID, 70}; +static const oid ednsPingMismatchesOID[] = {RECURSOR_STATS_OID, 71}; +static const oid dnssecQueriesOID[] = {RECURSOR_STATS_OID, 72}; +static const oid nopingOutqueriesOID[] = {RECURSOR_STATS_OID, 73}; +static const oid noednsOutqueriesOID[] = {RECURSOR_STATS_OID, 74}; +static const oid uptimeOID[] = {RECURSOR_STATS_OID, 75}; +static const oid realMemoryUsageOID[] = {RECURSOR_STATS_OID, 76}; +static const oid fdUsageOID[] = {RECURSOR_STATS_OID, 77}; +static const oid userMsecOID[] = {RECURSOR_STATS_OID, 78}; +static const oid sysMsecOID[] = {RECURSOR_STATS_OID, 79}; +static const oid dnssecValidationsOID[] = {RECURSOR_STATS_OID, 80}; +static const oid dnssecResultInsecureOID[] = {RECURSOR_STATS_OID, 81}; +static const oid dnssecResultSecureOID[] = {RECURSOR_STATS_OID, 82}; +static const oid dnssecResultBogusOID[] = {RECURSOR_STATS_OID, 83}; +static const oid dnssecResultIndeterminateOID[] = {RECURSOR_STATS_OID, 84}; +static const oid dnssecResultNtaOID[] = {RECURSOR_STATS_OID, 85}; +static const oid policyResultNoactionOID[] = {RECURSOR_STATS_OID, 86}; +static const oid policyResultDropOID[] = {RECURSOR_STATS_OID, 87}; +static const oid policyResultNxdomainOID[] = {RECURSOR_STATS_OID, 88}; +static const oid policyResultNodataOID[] = {RECURSOR_STATS_OID, 89}; +static const oid policyResultTruncateOID[] = {RECURSOR_STATS_OID, 90}; +static const oid policyResultCustomOID[] = {RECURSOR_STATS_OID, 91}; +static const oid queryPipeFullDropsOID[] = {RECURSOR_STATS_OID, 92}; +static const oid truncatedDropsOID[] = {RECURSOR_STATS_OID, 93}; +static const oid emptyQueriesOID[] = {RECURSOR_STATS_OID, 94}; +static const oid dnssecAuthenticDataQueriesOID[] = {RECURSOR_STATS_OID, 95}; +static const oid dnssecCheckDisabledQueriesOID[] = {RECURSOR_STATS_OID, 96}; +static const oid variableResponsesOID[] = {RECURSOR_STATS_OID, 97}; +static const oid specialMemoryUsageOID[] = {RECURSOR_STATS_OID, 98}; +static const oid rebalancedQueriesOID[] = {RECURSOR_STATS_OID, 99}; +static const oid qnameMinFallbackSuccessOID[] = {RECURSOR_STATS_OID, 100}; +static const oid proxyProtocolInvalidOID[] = {RECURSOR_STATS_OID, 101}; +static const oid recordCacheContendedOID[] = {RECURSOR_STATS_OID, 102}; +static const oid recordCacheAcquiredOID[] = {RECURSOR_STATS_OID, 103}; +static const oid nodLookupsDroppedOversizeOID[] = {RECURSOR_STATS_OID, 104}; +static const oid taskQueuePushedOID[] = {RECURSOR_STATS_OID, 105}; +static const oid taskQueueExpiredOID[] = {RECURSOR_STATS_OID, 106}; +static const oid taskQueueSizeOID[] = {RECURSOR_STATS_OID, 107}; +static const oid aggressiveNSECCacheEntriesOID[] = {RECURSOR_STATS_OID, 108}; +static const oid aggressiveNSECCacheNSECHitsOID[] = {RECURSOR_STATS_OID, 109}; +static const oid aggressiveNSECCacheNSEC3HitsOID[] = {RECURSOR_STATS_OID, 110}; +static const oid aggressiveNSECCacheNSECWCHitsOID[] = {RECURSOR_STATS_OID, 111}; +static const oid aggressiveNSECCacheNSEC3WCHitsOID[] = {RECURSOR_STATS_OID, 112}; +static const oid dotOutqueriesOID[] = {RECURSOR_STATS_OID, 113}; +static const oid dns64PrefixAnswers[] = {RECURSOR_STATS_OID, 114}; +static const oid almostExpiredPushed[] = {RECURSOR_STATS_OID, 115}; +static const oid almostExpiredRun[] = {RECURSOR_STATS_OID, 116}; +static const oid almostExpiredExceptions[] = {RECURSOR_STATS_OID, 117}; +#ifdef __linux__ +static const oid udpInCsumErrorsOID[] = {RECURSOR_STATS_OID, 118}; +static const oid udp6RecvbufErrorsOID[] = {RECURSOR_STATS_OID, 119}; +static const oid udp6SndbufErrorsOID[] = {RECURSOR_STATS_OID, 120}; +static const oid udp6NoportErrorsOID[] = {RECURSOR_STATS_OID, 121}; +static const oid udp6InErrorsOID[] = {RECURSOR_STATS_OID, 122}; +static const oid udp6InCsumErrorsOID[] = {RECURSOR_STATS_OID, 123}; +#endif /* __linux__ */ +static const oid sourceDisallowedNotifyOID[] = {RECURSOR_STATS_OID, 124}; +static const oid zoneDisallowedNotifyOID[] = {RECURSOR_STATS_OID, 125}; static std::unordered_map s_statsMap; @@ -164,7 +174,8 @@ static int handleCounter64Stats(netsnmp_mib_handler* handler, boost::optional value = getStatByName(it->second); if (value) { return RecursorSNMPAgent::setCounter64Value(requests, *value); - } else { + } + else { return RecursorSNMPAgent::setCounter64Value(requests, 0); } } @@ -188,19 +199,18 @@ static int handleDisabledCounter64Stats(netsnmp_mib_handler* handler, static void registerCounter64Stat(const std::string& name, const oid statOID[], size_t statOIDLength) { if (statOIDLength != OID_LENGTH(questionsOID)) { - g_log< #include #include +#include #include "pdnsexception.hh" @@ -21,71 +22,43 @@ std::atomic RecursorControlChannel::stop = false; RecursorControlChannel::RecursorControlChannel() { - d_fd=-1; - *d_local.sun_path=0; - d_local.sun_family=0; + d_fd = -1; + *d_local.sun_path = 0; + d_local.sun_family = 0; } -RecursorControlChannel::~RecursorControlChannel() +RecursorControlChannel::~RecursorControlChannel() { - if(d_fd > 0) + if (d_fd > 0) close(d_fd); - if(*d_local.sun_path) + if (*d_local.sun_path) unlink(d_local.sun_path); } -static void setSocketBuffer(int fd, int optname, uint32_t size) +int RecursorControlChannel::listen(const string& fname) { - uint32_t psize=0; - socklen_t len=sizeof(psize); + d_fd = socket(AF_UNIX, SOCK_STREAM, 0); + setCloseOnExec(d_fd); - if (getsockopt(fd, SOL_SOCKET, optname, (void*)&psize, &len)) - throw PDNSException("Unable to getsocket buffer size: "+stringerror()); + if (d_fd < 0) + throw PDNSException("Creating UNIX domain socket: " + stringerror()); - if (psize > size) - return; - - // failure to raise is not fatal - setsockopt(fd, SOL_SOCKET, optname, (const void*)&size, sizeof(size)); -} + int tmp = 1; + if (setsockopt(d_fd, SOL_SOCKET, SO_REUSEADDR, (char*)&tmp, sizeof tmp) < 0) + throw PDNSException("Setsockopt failed: " + stringerror()); + int err = unlink(fname.c_str()); + if (err < 0 && errno != ENOENT) + throw PDNSException("Can't remove (previous) controlsocket '" + fname + "': " + stringerror() + " (try --socket-dir)"); -static void setSocketReceiveBuffer(int fd, uint32_t size) -{ - setSocketBuffer(fd, SO_RCVBUF, size); -} + if (makeUNsockaddr(fname, &d_local)) + throw PDNSException("Unable to bind to controlsocket, path '" + fname + "' is not a valid UNIX socket path."); -static void setSocketSendBuffer(int fd, uint32_t size) -{ - setSocketBuffer(fd, SO_SNDBUF, size); -} - -int RecursorControlChannel::listen(const string& fname) -{ - d_fd=socket(AF_UNIX,SOCK_DGRAM,0); - setCloseOnExec(d_fd); - - if(d_fd < 0) - throw PDNSException("Creating UNIX domain socket: "+stringerror()); - - int tmp=1; - if(setsockopt(d_fd, SOL_SOCKET, SO_REUSEADDR,(char*)&tmp,sizeof tmp)<0) - throw PDNSException("Setsockopt failed: "+stringerror()); - - int err=unlink(fname.c_str()); - if(err < 0 && errno!=ENOENT) - throw PDNSException("Can't remove (previous) controlsocket '"+fname+"': "+stringerror() + " (try --socket-dir)"); - - if(makeUNsockaddr(fname, &d_local)) - throw PDNSException("Unable to bind to controlsocket, path '"+fname+"' is not a valid UNIX socket path."); - - if(bind(d_fd, (sockaddr*)&d_local,sizeof(d_local))<0) - throw PDNSException("Unable to bind to controlsocket '"+fname+"': "+stringerror()); - - // receive buf should be size of max datagram plus address size - setSocketReceiveBuffer(d_fd, 60 * 1024); - setSocketSendBuffer(d_fd, 64 * 1024); - + if (bind(d_fd, (sockaddr*)&d_local, sizeof(d_local)) < 0) + throw PDNSException("Unable to bind to controlsocket '" + fname + "': " + stringerror()); + if (::listen(d_fd, 0) == -1) { + throw PDNSException("Unable to listen on controlsocket '" + fname + "': " + stringerror()); + } return d_fd; } @@ -93,76 +66,51 @@ void RecursorControlChannel::connect(const string& path, const string& fname) { struct sockaddr_un remote; - d_fd=socket(AF_UNIX,SOCK_DGRAM,0); + d_fd = socket(AF_UNIX, SOCK_STREAM, 0); setCloseOnExec(d_fd); - if(d_fd < 0) - throw PDNSException("Creating UNIX domain socket: "+stringerror()); + if (d_fd < 0) + throw PDNSException("Creating UNIX domain socket: " + stringerror()); try { - int tmp=1; - if(setsockopt(d_fd, SOL_SOCKET, SO_REUSEADDR,(char*)&tmp,sizeof tmp)<0) - throw PDNSException("Setsockopt failed: "+stringerror()); - - string localname=path+"/lsockXXXXXX"; - *d_local.sun_path=0; - if (makeUNsockaddr(localname, &d_local)) - throw PDNSException("Unable to bind to local temporary file, path '"+localname+"' is not a valid UNIX socket path."); - - if(mkstemp(d_local.sun_path) < 0) - throw PDNSException("Unable to generate local temporary file in directory '"+path+"': "+stringerror()); - - int err=unlink(d_local.sun_path); - if(err < 0 && errno!=ENOENT) - throw PDNSException("Unable to remove local controlsocket: "+stringerror()); - - if(bind(d_fd, (sockaddr*)&d_local,sizeof(d_local))<0) - throw PDNSException("Unable to bind to local temporary file: "+stringerror()); + int tmp = 1; + if (setsockopt(d_fd, SOL_SOCKET, SO_REUSEADDR, (char*)&tmp, sizeof tmp) < 0) + throw PDNSException("Setsockopt failed: " + stringerror()); - if(chmod(d_local.sun_path,0666)<0) // make sure that pdns can reply! - throw PDNSException("Unable to chmod local temporary socket: "+stringerror()); - - string remotename=path+"/"+fname; + string remotename = path + "/" + fname; if (makeUNsockaddr(remotename, &remote)) - throw PDNSException("Unable to connect to controlsocket, path '"+remotename+"' is not a valid UNIX socket path."); + throw PDNSException("Unable to connect to controlsocket, path '" + remotename + "' is not a valid UNIX socket path."); - if(::connect(d_fd, (sockaddr*)&remote, sizeof(remote)) < 0) { - if(*d_local.sun_path) - unlink(d_local.sun_path); - throw PDNSException("Unable to connect to remote '"+string(remote.sun_path)+"': "+stringerror()); + if (::connect(d_fd, (sockaddr*)&remote, sizeof(remote)) < 0) { + if (*d_local.sun_path) + unlink(d_local.sun_path); + throw PDNSException("Unable to connect to remote '" + string(remote.sun_path) + "': " + stringerror()); } - - // receive buf should be size of max datagram plus address size - setSocketReceiveBuffer(d_fd, 60 * 1024); - setSocketSendBuffer(d_fd, 64 * 1024); - - } catch (...) { + } + catch (...) { close(d_fd); - d_fd=-1; - d_local.sun_path[0]=0; + d_fd = -1; + d_local.sun_path[0] = 0; throw; } } -static void sendfd(int s, int fd, const string* remote) +static void sendfd(int s, int fd) { - struct msghdr msg; - struct cmsghdr *cmsg; - union { + struct msghdr msg; + struct cmsghdr* cmsg; + union + { struct cmsghdr hdr; - unsigned char buf[CMSG_SPACE(sizeof(int))]; + unsigned char buf[CMSG_SPACE(sizeof(int))]; } cmsgbuf; struct iovec io_vector[1]; char ch = 'X'; - + io_vector[0].iov_base = &ch; io_vector[0].iov_len = 1; memset(&msg, 0, sizeof(msg)); - if (remote) { - msg.msg_name = const_cast(remote->c_str()); - msg.msg_namelen = remote->length(); - } msg.msg_control = &cmsgbuf.buf; msg.msg_controllen = sizeof(cmsgbuf.buf); msg.msg_iov = io_vector; @@ -172,70 +120,101 @@ static void sendfd(int s, int fd, const string* remote) cmsg->cmsg_len = CMSG_LEN(sizeof(int)); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; - *(int *)CMSG_DATA(cmsg) = fd; + *(int*)CMSG_DATA(cmsg) = fd; if (sendmsg(s, &msg, 0) == -1) { - throw PDNSException("Unable to send fd message over control channel: "+stringerror()); + throw PDNSException("Unable to send fd message over control channel: " + stringerror()); } } -void RecursorControlChannel::send(const Answer& msg, const std::string* remote, unsigned int timeout, int fd) +void RecursorControlChannel::send(int fd, const Answer& msg, unsigned int timeout, int fd_to_pass) { - int ret = waitForRWData(d_fd, false, timeout, 0); - if(ret == 0) { + int ret = waitForRWData(fd, false, timeout, 0); + if (ret == 0) { throw PDNSException("Timeout sending message over control channel"); } - else if(ret < 0) { + else if (ret < 0) { throw PDNSException("Error sending message over control channel:" + stringerror()); } - if(remote) { - struct sockaddr_un remoteaddr; - memset(&remoteaddr, 0, sizeof(remoteaddr)); - - remoteaddr.sun_family=AF_UNIX; - strncpy(remoteaddr.sun_path, remote->c_str(), sizeof(remoteaddr.sun_path)-1); - remoteaddr.sun_path[sizeof(remoteaddr.sun_path)-1] = '\0'; + if (::send(fd, &msg.d_ret, sizeof(msg.d_ret), 0) < 0) { + throw PDNSException("Unable to send return code over control channel: " + stringerror()); + } + size_t len = msg.d_str.length(); + if (::send(fd, &len, sizeof(len), 0) < 0) { + throw PDNSException("Unable to send length over control channel: " + stringerror()); + } + if (::send(fd, msg.d_str.c_str(), len, 0) != static_cast(len)) { + throw PDNSException("Unable to send message over control channel: " + stringerror()); + } - if(::sendto(d_fd, &msg.d_ret, sizeof(msg.d_ret), 0, (struct sockaddr*) &remoteaddr, sizeof(remoteaddr) ) < 0) - throw PDNSException("Unable to send message over control channel '"+string(remoteaddr.sun_path)+"': "+stringerror()); - if(::sendto(d_fd, msg.d_str.c_str(), msg.d_str.length(), 0, (struct sockaddr*) &remoteaddr, sizeof(remoteaddr) ) < 0) - throw PDNSException("Unable to send message over control channel '"+string(remoteaddr.sun_path)+"': "+stringerror()); + if (fd_to_pass != -1) { + sendfd(fd, fd_to_pass); } - else { - if(::send(d_fd, &msg.d_ret, sizeof(msg.d_ret), 0) < 0) - throw PDNSException("Unable to send message over control channel: "+stringerror()); - if(::send(d_fd, msg.d_str.c_str(), msg.d_str.length(), 0) < 0) - throw PDNSException("Unable to send message over control channel: "+stringerror()); +} + +static void waitForRead(int fd, unsigned int timeout, time_t start) +{ + time_t elapsed = time(nullptr) - start; + if (elapsed >= timeout) { + throw PDNSException("Timeout waiting for control channel data"); } - if (fd != -1) { - sendfd(d_fd, fd, remote); + int ret = waitForData(fd, timeout - elapsed, 0); + if (ret == 0) { + throw PDNSException("Timeout waiting for control channel data"); } } -RecursorControlChannel::Answer RecursorControlChannel::recv(std::string* remote, unsigned int timeout) +static size_t getArgMax() { - char buffer[16384]; - ssize_t len; - struct sockaddr_un remoteaddr; - socklen_t addrlen = sizeof(remoteaddr); +#if defined(ARG_MAX) + return ARG_MAX; +#endif - int ret = waitForData(d_fd, timeout, 0); - if (ret == 0) { - throw PDNSException("Timeout waiting for answer from control channel"); +#if defined(_SC_ARG_MAX) + auto tmp = sysconf(_SC_ARG_MAX); + if (tmp != -1) { + return tmp; } +#endif + /* _POSIX_ARG_MAX */ + return 4096; +} + +RecursorControlChannel::Answer RecursorControlChannel::recv(int fd, unsigned int timeout) +{ + // timeout covers the operation of all read ops combined + const time_t start = time(nullptr); + + waitForRead(fd, timeout, start); int err; - if (::recvfrom(d_fd, &err, sizeof(err), 0, (struct sockaddr*)&remoteaddr, &addrlen) != sizeof(err)) { - throw PDNSException("Unable to receive return status over control channel1: " + stringerror()); + if (::recv(fd, &err, sizeof(err), 0) != sizeof(err)) { + throw PDNSException("Unable to receive return status over control channel: " + stringerror()); } - if ((len = ::recvfrom(d_fd, buffer, sizeof(buffer), 0, (struct sockaddr*)&remoteaddr, &addrlen)) < 0) { - throw PDNSException("Unable to receive message over control channel2: "+stringerror()); + + waitForRead(fd, timeout, start); + size_t len; + if (::recv(fd, &len, sizeof(len), 0) != sizeof(len)) { + throw PDNSException("Unable to receive length over control channel: " + stringerror()); } - if(remote) { - *remote=remoteaddr.sun_path; + if (len > getArgMax()) { + throw PDNSException("Length of control channel message too large"); } - return {err, string(buffer, buffer + len)}; -} + string str; + str.reserve(len); + while (str.length() < len) { + char buffer[1024]; + waitForRead(fd, timeout, start); + size_t toRead = std::min(len - str.length(), sizeof(buffer)); + ssize_t recvd = ::recv(fd, buffer, toRead, 0); + if (recvd <= 0) { + // EOF means we have a length error + throw PDNSException("Unable to receive message over control channel: " + stringerror()); + } + str.append(buffer, recvd); + } + return {err, str}; +} diff --git a/pdns/rec_channel.hh b/pdns/rec_channel.hh index b89267aaaa90..14d5e392bc4a 100644 --- a/pdns/rec_channel.hh +++ b/pdns/rec_channel.hh @@ -65,8 +65,8 @@ public: std::string d_str; }; - void send(const Answer&, const std::string* remote = nullptr, unsigned int timeout = 5, int fd = -1); - RecursorControlChannel::Answer recv(std::string* remote = nullptr, unsigned int timeout = 5); + void send(int remote, const Answer&, unsigned int timeout = 5, int fd_to_pass = -1); + RecursorControlChannel::Answer recv(int fd, unsigned int timeout = 5); int d_fd; static std::atomic stop; @@ -81,16 +81,22 @@ public: RecursorControlParser() { } - static void nop(void){} + static void nop(void) {} typedef void func_t(void); RecursorControlChannel::Answer getAnswer(int s, const std::string& question, func_t** func); }; +enum class StatComponent +{ + API, + Carbon, + RecControl, + SNMP +}; -enum class StatComponent { API, Carbon, RecControl, SNMP }; - -struct StatsMapEntry { +struct StatsMapEntry +{ std::string d_prometheusName; std::string d_value; }; @@ -99,6 +105,7 @@ class PrefixDashNumberCompare { private: static std::pair prefixAndTrailingNum(const std::string& a); + public: bool operator()(const std::string& a, const std::string& b) const; }; @@ -117,9 +124,9 @@ struct CarbonConfig extern GlobalStateHolder g_carbonConfig; -std::vector >* pleaseGetQueryRing(); -std::vector >* pleaseGetServfailQueryRing(); -std::vector >* pleaseGetBogusQueryRing(); +std::vector>* pleaseGetQueryRing(); +std::vector>* pleaseGetServfailQueryRing(); +std::vector>* pleaseGetBogusQueryRing(); std::vector* pleaseGetRemotes(); std::vector* pleaseGetServfailRemotes(); std::vector* pleaseGetBogusRemotes(); diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc index 360b1f54c89b..c87c4cc56b50 100644 --- a/pdns/rec_channel_rec.cc +++ b/pdns/rec_channel_rec.cc @@ -39,16 +39,17 @@ #include "pubsuffix.hh" #include "namespaces.hh" #include "rec-taskqueue.hh" +#include "rec-tcpout.hh" std::pair PrefixDashNumberCompare::prefixAndTrailingNum(const std::string& a) { auto i = a.length(); if (i == 0) { - return make_pair(a, ""); + return {a, ""}; } --i; if (!std::isdigit(a[i])) { - return make_pair(a, ""); + return {a, ""}; } while (i > 0) { if (!std::isdigit(a[i])) { @@ -56,7 +57,7 @@ std::pair PrefixDashNumberCompare::prefixAndTrailingNu } --i; } - return make_pair(a.substr(0, i + 1), a.substr(i + 1, a.size() - i - 1)); + return {a.substr(0, i + 1), a.substr(i + 1, a.size() - i - 1)}; } bool PrefixDashNumberCompare::operator()(const std::string& a, const std::string& b) const @@ -73,12 +74,13 @@ bool PrefixDashNumberCompare::operator()(const std::string& a, const std::string } static map d_get32bitpointers; -static map*> d_getatomics; -static map> d_get64bitmembers; +static map d_getatomics; +static map> d_get64bitmembers; static map> d_getmultimembers; -struct dynmetrics { - std::atomic *d_ptr; +struct dynmetrics +{ + std::atomic* d_ptr; std::string d_prometheusName; }; @@ -101,7 +103,7 @@ void disableStats(StatComponent component, const string& stats) std::vector disabledStats; stringtok(disabledStats, stats, ", "); auto& map = s_disabledStats[component]; - for (const auto &st : disabledStats) { + for (const auto& st : disabledStats) { map.insert(st); } } @@ -111,7 +113,7 @@ static void addGetStat(const string& name, const uint32_t* place) d_get32bitpointers[name] = place; } -static void addGetStat(const string& name, const std::atomic* place) +static void addGetStat(const string& name, const pdns::stat_t* place) { d_getatomics[name] = place; } @@ -129,8 +131,8 @@ static void addGetStat(const string& name, std::function f) static std::string getPrometheusName(const std::string& arg) { std::string name = arg; - std::replace_if(name.begin(), name.end(), [](char c){ - return !isalnum(static_cast(c));}, '_'); + std::replace_if( + name.begin(), name.end(), [](char c) { return !isalnum(static_cast(c)); }, '_'); return "pdns_recursor_" + name; } @@ -145,12 +147,13 @@ std::atomic* getDynMetric(const std::string& str, const std::stri std::string name(str); if (!prometheusName.empty()) { name = prometheusName; - } else { + } + else { name = getPrometheusName(name); } auto ret = dynmetrics{new std::atomic(), name}; - (*dm)[str]= ret; + (*dm)[str] = ret; return ret.d_ptr; } @@ -158,11 +161,11 @@ static boost::optional get(const string& name) { boost::optional ret; - if(d_get32bitpointers.count(name)) + if (d_get32bitpointers.count(name)) return *d_get32bitpointers.find(name)->second; - if(d_getatomics.count(name)) + if (d_getatomics.count(name)) return d_getatomics.find(name)->second->load(); - if(d_get64bitmembers.count(name)) + if (d_get64bitmembers.count(name)) return d_get64bitmembers.find(name)->second(); { @@ -173,7 +176,7 @@ static boost::optional get(const string& name) } } - for(const auto& themultimember : d_getmultimembers) { + for (const auto& themultimember : d_getmultimembers) { const auto items = themultimember.second(); const auto item = items.find(name); if (item != items.end()) { @@ -194,33 +197,33 @@ StatsMap getAllStatsMap(StatComponent component) StatsMap ret; const auto& disabledlistMap = s_disabledStats.at(component); - for(const auto& the32bits : d_get32bitpointers) { + for (const auto& the32bits : d_get32bitpointers) { if (disabledlistMap.count(the32bits.first) == 0) { - ret.insert(make_pair(the32bits.first, StatsMapEntry{getPrometheusName(the32bits.first), std::to_string(*the32bits.second)})); + ret.emplace(the32bits.first, StatsMapEntry{getPrometheusName(the32bits.first), std::to_string(*the32bits.second)}); } } - for(const auto& atomic : d_getatomics) { + for (const auto& atomic : d_getatomics) { if (disabledlistMap.count(atomic.first) == 0) { - ret.insert(make_pair(atomic.first, StatsMapEntry{getPrometheusName(atomic.first), std::to_string(atomic.second->load())})); + ret.emplace(atomic.first, StatsMapEntry{getPrometheusName(atomic.first), std::to_string(atomic.second->load())}); } } - for(const auto& the64bitmembers : d_get64bitmembers) { + for (const auto& the64bitmembers : d_get64bitmembers) { if (disabledlistMap.count(the64bitmembers.first) == 0) { - ret.insert(make_pair(the64bitmembers.first, StatsMapEntry{getPrometheusName(the64bitmembers.first), std::to_string(the64bitmembers.second())})); + ret.emplace(the64bitmembers.first, StatsMapEntry{getPrometheusName(the64bitmembers.first), std::to_string(the64bitmembers.second())}); } } - for(const auto& themultimember : d_getmultimembers) { + for (const auto& themultimember : d_getmultimembers) { if (disabledlistMap.count(themultimember.first) == 0) { ret.merge(themultimember.second()); } } { - for(const auto& a : *(d_dynmetrics.lock())) { + for (const auto& a : *(d_dynmetrics.lock())) { if (disabledlistMap.count(a.first) == 0) { - ret.insert(make_pair(a.first, StatsMapEntry{a.second.d_prometheusName, std::to_string(*a.second.d_ptr)})); + ret.emplace(a.first, StatsMapEntry{a.second.d_prometheusName, std::to_string(*a.second.d_ptr)}); } } } @@ -238,73 +241,52 @@ static string getAllStats() return ret; } -template +template static string doGet(T begin, T end) { string ret; - for(T i=begin; i != end; ++i) { - boost::optional num=get(*i); - if(num) - ret+=std::to_string(*num)+"\n"; + for (T i = begin; i != end; ++i) { + boost::optional num = get(*i); + if (num) + ret += std::to_string(*num) + "\n"; else - ret+="UNKNOWN\n"; + ret += "UNKNOWN\n"; } return ret; } -template +template string static doGetParameter(T begin, T end) { string ret; string parm; using boost::replace_all; - for(T i=begin; i != end; ++i) { - if(::arg().parmIsset(*i)) { - parm=::arg()[*i]; + for (T i = begin; i != end; ++i) { + if (::arg().parmIsset(*i)) { + parm = ::arg()[*i]; replace_all(parm, "\\", "\\\\"); replace_all(parm, "\"", "\\\""); replace_all(parm, "\n", "\\n"); - ret += *i +"=\""+ parm +"\"\n"; + ret += *i + "=\"" + parm + "\"\n"; } else - ret += *i +" not known\n"; + ret += *i + " not known\n"; } return ret; } -struct FDWrapper : public boost::noncopyable -{ - FDWrapper(int descr) : fd(descr) {} - ~FDWrapper() - { - if (fd != -1) { - close(fd); - } - fd = -1; - } - FDWrapper(FDWrapper&& rhs) : fd(rhs.fd) - { - rhs.fd = -1; - } - operator int() const - { - return fd; - } -private: - int fd; -}; - /* Read an (open) fd from the control channel */ static FDWrapper getfd(int s) { int fd = -1; - struct msghdr msg; - struct cmsghdr *cmsg; - union { + struct msghdr msg; + struct cmsghdr* cmsg; + union + { struct cmsghdr hdr; - unsigned char buf[CMSG_SPACE(sizeof(int))]; + unsigned char buf[CMSG_SPACE(sizeof(int))]; } cmsgbuf; struct iovec io_vector[1]; char ch; @@ -326,24 +308,21 @@ getfd(int s) } for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) { - if (cmsg->cmsg_len == CMSG_LEN(sizeof(int)) && - cmsg->cmsg_level == SOL_SOCKET && - cmsg->cmsg_type == SCM_RIGHTS) { - fd = *(int *)CMSG_DATA(cmsg); + if (cmsg->cmsg_len == CMSG_LEN(sizeof(int)) && cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { + fd = *(int*)CMSG_DATA(cmsg); break; } } return FDWrapper(fd); } - static uint64_t dumpNegCache(int fd) { int newfd = dup(fd); if (newfd == -1) { return 0; } - auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); + auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); if (!fp) { return 0; } @@ -364,7 +343,7 @@ static uint64_t dumpAggressiveNSECCache(int fd) if (newfd == -1) { return 0; } - auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); + auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); if (!fp) { return 0; } @@ -411,24 +390,22 @@ static RecursorControlChannel::Answer doDumpToFile(int s, uint64_t* (*function)( auto fdw = getfd(s); if (fdw < 0) { - return { 1, name + ": error opening dump file for writing: " + stringerror() + "\n" }; + return {1, name + ": error opening dump file for writing: " + stringerror() + "\n"}; } uint64_t total = 0; try { int fd = fdw; - total = broadcastAccFunction([function, fd]{ return function(fd); }); + total = broadcastAccFunction([function, fd] { return function(fd); }); } - catch(std::exception& e) - { - return { 1, name + ": error dumping data: " + string(e.what()) + "\n" }; + catch (std::exception& e) { + return {1, name + ": error dumping data: " + string(e.what()) + "\n"}; } - catch(PDNSException& e) - { - return { 1, name + ": error dumping data: " + e.reason + "\n" }; + catch (PDNSException& e) { + return {1, name + ": error dumping data: " + e.reason + "\n"}; } - return { 0, name + ": dumped " + std::to_string(total) + " records\n" }; + return {0, name + ": dumped " + std::to_string(total) + " records\n"}; } // Does not follow the generic dump to file pattern, has a more complex lambda @@ -437,46 +414,46 @@ static RecursorControlChannel::Answer doDumpCache(int s) auto fdw = getfd(s); if (fdw < 0) { - return { 1, "Error opening dump file for writing: " + stringerror() + "\n" }; + return {1, "Error opening dump file for writing: " + stringerror() + "\n"}; } uint64_t total = 0; try { int fd = fdw; - total = g_recCache->doDump(fd) + dumpNegCache(fd) + broadcastAccFunction([fd]{ return pleaseDump(fd); }) + dumpAggressiveNSECCache(fd); + total = g_recCache->doDump(fd) + dumpNegCache(fd) + broadcastAccFunction([fd] { return pleaseDump(fd); }) + dumpAggressiveNSECCache(fd); + } + catch (...) { } - catch(...){} - return { 0, "dumped " + std::to_string(total) + " records\n" }; + return {0, "dumped " + std::to_string(total) + " records\n"}; } // Does not follow the generic dump to file pattern, has an argument -template +template static RecursorControlChannel::Answer doDumpRPZ(int s, T begin, T end) { auto fdw = getfd(s); if (fdw < 0) { - return { 1, "Error opening dump file for writing: " + stringerror() + "\n" }; + return {1, "Error opening dump file for writing: " + stringerror() + "\n"}; } T i = begin; if (i == end) { - return { 1, "No zone name specified\n" }; + return {1, "No zone name specified\n"}; } string zoneName = *i; auto luaconf = g_luaconfs.getLocal(); const auto zone = luaconf->dfe.getZone(zoneName); if (!zone) { - return { 1, "No RPZ zone named " + zoneName + "\n" }; + return {1, "No RPZ zone named " + zoneName + "\n"}; } - - auto fp = std::unique_ptr(fdopen(fdw, "w"), fclose); + auto fp = std::unique_ptr(fdopen(fdw, "w"), fclose); if (!fp) { int err = errno; - return { 1, "converting file descriptor: " + stringerror(err) + "\n" }; + return {1, "converting file descriptor: " + stringerror(err) + "\n"}; } zone->dump(fp.get()); @@ -484,51 +461,46 @@ static RecursorControlChannel::Answer doDumpRPZ(int s, T begin, T end) return {0, "done\n"}; } -uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree, uint16_t qtype) -{ - return new uint64_t(t_packetCache->doWipePacketCache(canon, qtype, subtree)); -} - -template +template static string doWipeCache(T begin, T end, uint16_t qtype) { - vector > toWipe; - for(T i=begin; i != end; ++i) { + vector> toWipe; + for (T i = begin; i != end; ++i) { DNSName canon; - bool subtree=false; + bool subtree = false; try { - if(boost::ends_with(*i, "$")) { - canon=DNSName(i->substr(0, i->size()-1)); - subtree=true; - } else { - canon=DNSName(*i); + if (boost::ends_with(*i, "$")) { + canon = DNSName(i->substr(0, i->size() - 1)); + subtree = true; + } + else { + canon = DNSName(*i); } - } catch (std::exception &e) { + } + catch (std::exception& e) { return "Error: " + std::string(e.what()) + ", nothing wiped\n"; } - toWipe.push_back({canon, subtree}); + toWipe.emplace_back(canon, subtree); } - int count=0, pcount=0, countNeg=0; + int count = 0, pcount = 0, countNeg = 0; for (auto wipe : toWipe) { try { - count += g_recCache->doWipeCache(wipe.first, wipe.second, qtype); - pcount += broadcastAccFunction([=]{ return pleaseWipePacketCache(wipe.first, wipe.second, qtype);}); - countNeg += g_negCache->wipe(wipe.first, wipe.second); - if (g_aggressiveNSECCache) { - g_aggressiveNSECCache->removeZoneInfo(wipe.first, wipe.second); - } + auto res = wipeCaches(wipe.first, wipe.second, qtype); + count += res.record_count; + pcount += res.packet_count; + countNeg += res.negative_record_count; } catch (const std::exception& e) { - g_log< +template static string doSetCarbonServer(T begin, T end) { auto config = g_carbonConfig.getCopy(); @@ -546,7 +518,8 @@ static string doSetCarbonServer(T begin, T end) if (begin != end) { config.hostname = *begin; ret += "set carbon-ourname to '" + *begin + "'\n"; - } else { + } + else { g_carbonConfig.setState(std::move(config)); return ret; } @@ -555,7 +528,8 @@ static string doSetCarbonServer(T begin, T end) if (begin != end) { config.namespace_name = *begin; ret += "set carbon-namespace to '" + *begin + "'\n"; - } else { + } + else { g_carbonConfig.setState(std::move(config)); return ret; } @@ -570,10 +544,10 @@ static string doSetCarbonServer(T begin, T end) return ret; } -template +template static string doSetDnssecLogBogus(T begin, T end) { - if(checkDNSSECDisabled()) + if (checkDNSSECDisabled()) return "DNSSEC is disabled in the configuration, not changing the Bogus logging setting\n"; if (begin == end) @@ -581,7 +555,7 @@ static string doSetDnssecLogBogus(T begin, T end) if (pdns_iequals(*begin, "on") || pdns_iequals(*begin, "yes")) { if (!g_dnssecLogBogus) { - g_log< +template static string doAddNTA(T begin, T end) { - if(checkDNSSECDisabled()) + if (checkDNSSECDisabled()) return "DNSSEC is disabled in the configuration, not adding a Negative Trust Anchor\n"; - if(begin == end) + if (begin == end) return "No NTA specified, doing nothing\n"; DNSName who; try { who = DNSName(*begin); } - catch(std::exception &e) { + catch (std::exception& e) { string ret("Can't add Negative Trust Anchor: "); ret += e.what(); ret += "\n"; @@ -628,39 +602,34 @@ static string doAddNTA(T begin, T end) if (begin != end) why += " "; } - g_log<doWipeCache(who, true, 0xffff); - broadcastAccFunction([=]{return pleaseWipePacketCache(who, true, 0xffff);}); - g_negCache->wipe(who, true); - if (g_aggressiveNSECCache) { - g_aggressiveNSECCache->removeZoneInfo(who, true); - } + wipeCaches(who, true, 0xffff); } catch (std::exception& e) { - g_log< +template static string doClearNTA(T begin, T end) { - if(checkDNSSECDisabled()) + if (checkDNSSECDisabled()) return "DNSSEC is disabled in the configuration, not removing a Negative Trust Anchor\n"; - if(begin == end) + if (begin == end) return "No Negative Trust Anchor specified, doing nothing.\n"; - if (begin + 1 == end && *begin == "*"){ - g_log<doWipeCache(entry, true, 0xffff); - broadcastAccFunction([=]{return pleaseWipePacketCache(entry, true, 0xffff);}); - g_negCache->wipe(entry, true); - if (g_aggressiveNSECCache) { - g_aggressiveNSECCache->removeZoneInfo(entry, true); - } + lci.negAnchors.erase(entry); + }); + wipeCaches(entry, true, 0xffff); if (!first) { first = false; removed += ","; @@ -703,8 +667,8 @@ static string doClearNTA(T begin, T end) removed += " " + entry.toStringRootDot(); } } - catch(std::exception &e) { - g_log< +template static string doAddTA(T begin, T end) { - if(checkDNSSECDisabled()) + if (checkDNSSECDisabled()) return "DNSSEC is disabled in the configuration, not adding a Trust Anchor\n"; - if(begin == end) + if (begin == end) return "No TA specified, doing nothing\n"; DNSName who; try { who = DNSName(*begin); } - catch(std::exception &e) { + catch (std::exception& e) { string ret("Can't add Trust Anchor: "); ret += e.what(); ret += "\n"; @@ -751,33 +715,28 @@ static string doAddTA(T begin, T end) } try { - g_log<(DSRecordContent::make(what)); + auto ds = std::dynamic_pointer_cast(DSRecordContent::make(what)); lci.dsAnchors[who].insert(*ds); - }); - g_recCache->doWipeCache(who, true, 0xffff); - broadcastAccFunction([=]{return pleaseWipePacketCache(who, true, 0xffff);}); - g_negCache->wipe(who, true); - if (g_aggressiveNSECCache) { - g_aggressiveNSECCache->removeZoneInfo(who, true); - } - g_log< +template static string doClearTA(T begin, T end) { - if(checkDNSSECDisabled()) + if (checkDNSSECDisabled()) return "DNSSEC is disabled in the configuration, not removing a Trust Anchor\n"; - if(begin == end) + if (begin == end) return "No Trust Anchor to clear\n"; vector toRemove; @@ -786,7 +745,7 @@ static string doClearTA(T begin, T end) try { who = DNSName(*begin); } - catch(std::exception &e) { + catch (std::exception& e) { string ret("Error: "); ret += e.what(); ret += ". No Anchors removed\n"; @@ -801,17 +760,12 @@ static string doClearTA(T begin, T end) string removed(""); bool first(true); try { - for (auto const &entry : toRemove) { - g_log<doWipeCache(entry, true, 0xffff); - broadcastAccFunction([=]{return pleaseWipePacketCache(entry, true, 0xffff);}); - g_negCache->wipe(entry, true); - if (g_aggressiveNSECCache) { - g_aggressiveNSECCache->removeZoneInfo(entry, true); - } + lci.dsAnchors.erase(entry); + }); + wipeCaches(entry, true, 0xffff); if (!first) { first = false; removed += ","; @@ -820,7 +774,7 @@ static string doClearTA(T begin, T end) } } catch (std::exception& e) { - g_log<dsAnchors) { ret += anchor.first.toLogString() + "\n"; for (auto e : anchor.second) { - ret+="\t\t"+e.getZoneRepresentation() + "\n"; + ret += "\t\t" + e.getZoneRepresentation() + "\n"; } } return ret; } -template +template static string setMinimumTTL(T begin, T end) { - if(end-begin != 1) + if (end - begin != 1) return "Need to supply new minimum TTL number\n"; try { SyncRes::s_minimumTTL = pdns_stou(*begin); @@ -858,10 +812,10 @@ static string setMinimumTTL(T begin, T end) } } -template +template static string setMinimumECSTTL(T begin, T end) { - if(end-begin != 1) + if (end - begin != 1) return "Need to supply new ECS minimum TTL number\n"; try { SyncRes::s_minimumECSTTL = pdns_stou(*begin); @@ -872,10 +826,10 @@ static string setMinimumECSTTL(T begin, T end) } } -template +template static string setMaxCacheEntries(T begin, T end) { - if(end-begin != 1) + if (end - begin != 1) return "Need to supply new cache size\n"; try { g_maxCacheEntries = pdns_stou(*begin); @@ -886,10 +840,10 @@ static string setMaxCacheEntries(T begin, T end) } } -template +template static string setMaxPacketCacheEntries(T begin, T end) { - if(end-begin != 1) + if (end - begin != 1) return "Need to supply new packet cache size\n"; try { g_maxPacketCacheEntries = pdns_stou(*begin); @@ -900,19 +854,18 @@ static string setMaxPacketCacheEntries(T begin, T end) } } - static uint64_t getSysTimeMsec() { struct rusage ru; getrusage(RUSAGE_SELF, &ru); - return (ru.ru_stime.tv_sec*1000ULL + ru.ru_stime.tv_usec/1000); + return (ru.ru_stime.tv_sec * 1000ULL + ru.ru_stime.tv_usec / 1000); } static uint64_t getUserTimeMsec() { struct rusage ru; getrusage(RUSAGE_SELF, &ru); - return (ru.ru_utime.tv_sec*1000ULL + ru.ru_utime.tv_usec/1000); + return (ru.ru_utime.tv_sec * 1000ULL + ru.ru_utime.tv_usec / 1000); } /* This is a pretty weird set of functions. To get per-thread cpu usage numbers, @@ -926,12 +879,12 @@ static uint64_t getUserTimeMsec() static ThreadTimes* pleaseGetThreadCPUMsec() { - uint64_t ret=0; + uint64_t ret = 0; #ifdef RUSAGE_THREAD struct rusage ru; getrusage(RUSAGE_THREAD, &ru); - ret = (ru.ru_utime.tv_sec*1000ULL + ru.ru_utime.tv_usec/1000); - ret += (ru.ru_stime.tv_sec*1000ULL + ru.ru_stime.tv_usec/1000); + ret = (ru.ru_utime.tv_sec * 1000ULL + ru.ru_utime.tv_usec / 1000); + ret += (ru.ru_stime.tv_sec * 1000ULL + ru.ru_stime.tv_usec / 1000); #endif return new ThreadTimes{ret, vector()}; } @@ -949,9 +902,9 @@ static uint64_t doGetThreadCPUMsec(int n) static ThreadTimes tt; std::lock_guard l(s_mut); - if(last != time(nullptr)) { - tt = broadcastAccFunction(pleaseGetThreadCPUMsec); - last = time(nullptr); + if (last != time(nullptr)) { + tt = broadcastAccFunction(pleaseGetThreadCPUMsec); + last = time(nullptr); } return tt.times.at(n); @@ -968,26 +921,25 @@ static string* pleaseGetCurrentQueries() struct timeval now; gettimeofday(&now, 0); - ostr << getMT()->d_waiters.size() <<" currently outstanding questions\n"; + ostr << getMT()->d_waiters.size() << " currently outstanding questions\n"; boost::format fmt("%1% %|40t|%2% %|47t|%3% %|63t|%4% %|68t|%5% %|78t|%6%\n"); ostr << (fmt % "qname" % "qtype" % "remote" % "tcp" % "chained" % "spent(ms)"); - unsigned int n=0; - for(const auto& mthread : getMT()->d_waiters) { + unsigned int n = 0; + for (const auto& mthread : getMT()->d_waiters) { const std::shared_ptr& pident = mthread.key; const double spent = g_networkTimeoutMsec - (DiffTime(now, mthread.ttd) * 1000); - ostr << (fmt - % pident->domain.toLogString() /* ?? */ % DNSRecordContent::NumberToType(pident->type) + ostr << (fmt + % pident->domain.toLogString() /* ?? */ % DNSRecordContent::NumberToType(pident->type) % pident->remote.toString() % (pident->tcpsock ? 'Y' : 'n') % (pident->fd == -1 ? 'Y' : 'n') - % (spent > 0 ? spent : '0') - ); + % (spent > 0 ? spent : '0')); ++n; if (n >= 100) break; } - ostr <<" - done\n"; + ostr << " - done\n"; return new string(ostr.str()); } @@ -1013,7 +965,7 @@ static uint64_t getNegCacheSize() static uint64_t* pleaseGetFailedHostsSize() { - uint64_t tmp=(SyncRes::getThrottledServersSize()); + uint64_t tmp = (SyncRes::getThrottledServersSize()); return new uint64_t(tmp); } @@ -1115,7 +1067,7 @@ static uint64_t doGetPacketCacheMisses() static uint64_t doGetMallocated() { // this turned out to be broken -/* struct mallinfo mi = mallinfo(); + /* struct mallinfo mi = mallinfo(); return mi.uordblks; */ return 0; } @@ -1129,14 +1081,43 @@ static StatsMap toStatsMap(const string& name, const pdns::AtomicHistogram& hist for (const auto& bucket : data) { snprintf(buf, sizeof(buf), "%g", bucket.d_boundary / 1e6); - std::string pname = pbasename + "seconds_bucket{" + "le=\"" + - (bucket.d_boundary == std::numeric_limits::max() ? "+Inf" : buf) + "\"}"; - entries.emplace(make_pair(bucket.d_name, StatsMapEntry{pname, std::to_string(bucket.d_count)})); + std::string pname = pbasename + "seconds_bucket{" + "le=\"" + (bucket.d_boundary == std::numeric_limits::max() ? "+Inf" : buf) + "\"}"; + entries.emplace(bucket.d_name, StatsMapEntry{pname, std::to_string(bucket.d_count)}); } snprintf(buf, sizeof(buf), "%g", histogram.getSum() / 1e6); - entries.emplace(make_pair(name + "sum", StatsMapEntry{pbasename + "seconds_sum", buf})); - entries.emplace(make_pair(name + "count", StatsMapEntry{pbasename + "seconds_count", std::to_string(data.back().d_count)})); + entries.emplace(name + "sum", StatsMapEntry{pbasename + "seconds_sum", buf}); + entries.emplace(name + "count", StatsMapEntry{pbasename + "seconds_count", std::to_string(data.back().d_count)}); + + return entries; +} + +static StatsMap toStatsMap(const string& name, const pdns::AtomicHistogram& histogram4, const pdns::AtomicHistogram& histogram6) +{ + const string pbasename = getPrometheusName(name); + StatsMap entries; + char buf[32]; + std::string pname; + + const auto& data4 = histogram4.getCumulativeBuckets(); + for (const auto& bucket : data4) { + snprintf(buf, sizeof(buf), "%g", bucket.d_boundary / 1e6); + pname = pbasename + "seconds_bucket{ipversion=\"v4\",le=\"" + (bucket.d_boundary == std::numeric_limits::max() ? "+Inf" : buf) + "\"}"; + entries.emplace(bucket.d_name + "4", StatsMapEntry{pname, std::to_string(bucket.d_count)}); + } + snprintf(buf, sizeof(buf), "%g", histogram4.getSum() / 1e6); + entries.emplace(name + "sum4", StatsMapEntry{pbasename + "seconds_sum{ipversion=\"v4\"}", buf}); + entries.emplace(name + "count4", StatsMapEntry{pbasename + "seconds_count{ipversion=\"v4\"}", std::to_string(data4.back().d_count)}); + + const auto& data6 = histogram6.getCumulativeBuckets(); + for (const auto& bucket : data6) { + snprintf(buf, sizeof(buf), "%g", bucket.d_boundary / 1e6); + pname = pbasename + "seconds_bucket{ipversion=\"v6\",le=\"" + (bucket.d_boundary == std::numeric_limits::max() ? "+Inf" : buf) + "\"}"; + entries.emplace(bucket.d_name + "6", StatsMapEntry{pname, std::to_string(bucket.d_count)}); + } + snprintf(buf, sizeof(buf), "%g", histogram6.getSum() / 1e6); + entries.emplace(name + "sum6", StatsMapEntry{pbasename + "seconds_sum{ipversion=\"v6\"}", buf}); + entries.emplace(name + "count6", StatsMapEntry{pbasename + "seconds_count{ipversion=\"v6\"}", std::to_string(data6.back().d_count)}); return entries; } @@ -1147,26 +1128,27 @@ static StatsMap toCPUStatsMap(const string& name) StatsMap entries; for (unsigned int n = 0; n < g_numThreads; ++n) { uint64_t tm = doGetThreadCPUMsec(n); - std::string pname = pbasename + "{thread=" + std::to_string(n) + '}'; - entries.emplace(make_pair(name + "-thread-" + std::to_string(n), StatsMapEntry{pname, std::to_string(tm)})); + std::string pname = pbasename + "{thread=\"" + std::to_string(n) + "\"}"; + entries.emplace(name + "-thread-" + std::to_string(n), StatsMapEntry{pname, std::to_string(tm)}); } return entries; } -static StatsMap toRPZStatsMap(const string& name, LockGuarded>>& map) +static StatsMap toRPZStatsMap(const string& name, LockGuarded>& map) { const string pbasename = getPrometheusName(name); StatsMap entries; uint64_t total = 0; - for (const auto& entry: *map.lock()) { - auto &key = entry.first; + for (const auto& entry : *map.lock()) { + auto& key = entry.first; auto count = entry.second.load(); std::string sname, pname; if (key.empty()) { sname = name + "-filter"; pname = pbasename + "{type=\"filter\"}"; - } else { + } + else { sname = name + "-rpz-" + key; pname = pbasename + "{type=\"rpz\",policyname=\"" + key + "\"}"; } @@ -1186,33 +1168,35 @@ static void registerAllStats1() addGetStat("tcp-questions", &g_stats.tcpqcounter); addGetStat("cache-hits", doGetCacheHits); - addGetStat("cache-misses", doGetCacheMisses); + addGetStat("cache-misses", doGetCacheMisses); addGetStat("cache-entries", doGetCacheSize); addGetStat("max-cache-entries", []() { return g_maxCacheEntries.load(); }); - addGetStat("max-packetcache-entries", []() { return g_maxPacketCacheEntries.load();}); - addGetStat("cache-bytes", doGetCacheBytes); - addGetStat("record-cache-contended", []() { return g_recCache->stats().first;}); - addGetStat("record-cache-acquired", []() { return g_recCache->stats().second;}); - + addGetStat("max-packetcache-entries", []() { return g_maxPacketCacheEntries.load(); }); + addGetStat("cache-bytes", doGetCacheBytes); + addGetStat("record-cache-contended", []() { return g_recCache->stats().first; }); + addGetStat("record-cache-acquired", []() { return g_recCache->stats().second; }); + addGetStat("packetcache-hits", doGetPacketCacheHits); - addGetStat("packetcache-misses", doGetPacketCacheMisses); - addGetStat("packetcache-entries", doGetPacketCacheSize); - addGetStat("packetcache-bytes", doGetPacketCacheBytes); + addGetStat("packetcache-misses", doGetPacketCacheMisses); + addGetStat("packetcache-entries", doGetPacketCacheSize); + addGetStat("packetcache-bytes", doGetPacketCacheBytes); - addGetStat("aggressive-nsec-cache-entries", [](){ return g_aggressiveNSECCache ? g_aggressiveNSECCache->getEntriesCount() : 0; }); - addGetStat("aggressive-nsec-cache-nsec-hits", [](){ return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSECHits() : 0; }); - addGetStat("aggressive-nsec-cache-nsec3-hits", [](){ return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSEC3Hits() : 0; }); - addGetStat("aggressive-nsec-cache-nsec-wc-hits", [](){ return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSECWildcardHits() : 0; }); - addGetStat("aggressive-nsec-cache-nsec3-wc-hits", [](){ return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSEC3WildcardHits() : 0; }); + addGetStat("aggressive-nsec-cache-entries", []() { return g_aggressiveNSECCache ? g_aggressiveNSECCache->getEntriesCount() : 0; }); + addGetStat("aggressive-nsec-cache-nsec-hits", []() { return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSECHits() : 0; }); + addGetStat("aggressive-nsec-cache-nsec3-hits", []() { return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSEC3Hits() : 0; }); + addGetStat("aggressive-nsec-cache-nsec-wc-hits", []() { return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSECWildcardHits() : 0; }); + addGetStat("aggressive-nsec-cache-nsec3-wc-hits", []() { return g_aggressiveNSECCache ? g_aggressiveNSECCache->getNSEC3WildcardHits() : 0; }); addGetStat("malloc-bytes", doGetMallocated); - + addGetStat("servfail-answers", &g_stats.servFails); addGetStat("nxdomain-answers", &g_stats.nxDomains); addGetStat("noerror-answers", &g_stats.noErrors); addGetStat("unauthorized-udp", &g_stats.unauthorizedUDP); addGetStat("unauthorized-tcp", &g_stats.unauthorizedTCP); + addGetStat("source-disallowed-notify", &g_stats.sourceDisallowedNotify); + addGetStat("zone-disallowed-notify", &g_stats.zoneDisallowedNotify); addGetStat("tcp-client-overflow", &g_stats.tcpClientOverflow); addGetStat("client-parse-errors", &g_stats.clientParseError); @@ -1287,13 +1271,19 @@ static void registerAllStats1() addGetStat("ecs-queries", &SyncRes::s_ecsqueries); addGetStat("ecs-responses", &SyncRes::s_ecsresponses); addGetStat("chain-resends", &g_stats.chainResends); - addGetStat("tcp-clients", []{return TCPConnection::getCurrentConnections();}); + addGetStat("tcp-clients", [] { return TCPConnection::getCurrentConnections(); }); #ifdef __linux__ - addGetStat("udp-recvbuf-errors", []{return udpErrorStats("udp-recvbuf-errors");}); - addGetStat("udp-sndbuf-errors", []{return udpErrorStats("udp-sndbuf-errors");}); - addGetStat("udp-noport-errors", []{return udpErrorStats("udp-noport-errors");}); - addGetStat("udp-in-errors", []{return udpErrorStats("udp-in-errors");}); + addGetStat("udp-recvbuf-errors", [] { return udpErrorStats("udp-recvbuf-errors"); }); + addGetStat("udp-sndbuf-errors", [] { return udpErrorStats("udp-sndbuf-errors"); }); + addGetStat("udp-noport-errors", [] { return udpErrorStats("udp-noport-errors"); }); + addGetStat("udp-in-errors", [] { return udpErrorStats("udp-in-errors"); }); + addGetStat("udp-in-csum-errors", [] { return udpErrorStats("udp-in-csum-errors"); }); + addGetStat("udp6-recvbuf-errors", [] { return udp6ErrorStats("udp6-recvbuf-errors"); }); + addGetStat("udp6-sndbuf-errors", [] { return udp6ErrorStats("udp6-sndbuf-errors"); }); + addGetStat("udp6-noport-errors", [] { return udp6ErrorStats("udp6-noport-errors"); }); + addGetStat("udp6-in-errors", [] { return udp6ErrorStats("udp6-in-errors"); }); + addGetStat("udp6-in-csum-errors", [] { return udp6ErrorStats("udp6-in-csum-errors"); }); #endif addGetStat("edns-ping-matches", &g_stats.ednsPingMatches); @@ -1309,32 +1299,32 @@ static void registerAllStats1() addGetStat("noedns-outqueries", &g_stats.noEdnsOutQueries); addGetStat("uptime", calculateUptime); - addGetStat("real-memory-usage", []{ return getRealMemoryUsage(string()); }); - addGetStat("special-memory-usage", []{ return getSpecialMemoryUsage(string()); }); - addGetStat("fd-usage", []{ return getOpenFileDescriptors(string()); }); + addGetStat("real-memory-usage", [] { return getRealMemoryUsage(string()); }); + addGetStat("special-memory-usage", [] { return getSpecialMemoryUsage(string()); }); + addGetStat("fd-usage", [] { return getOpenFileDescriptors(string()); }); // addGetStat("query-rate", getQueryRate); addGetStat("user-msec", getUserTimeMsec); addGetStat("sys-msec", getSysTimeMsec); #ifdef __linux__ - addGetStat("cpu-iowait", []{ return getCPUIOWait(string()); }); - addGetStat("cpu-steal", []{ return getCPUSteal(string()); }); + addGetStat("cpu-iowait", [] { return getCPUIOWait(string()); }); + addGetStat("cpu-steal", [] { return getCPUSteal(string()); }); #endif addGetStat("cpu-msec", []() { return toCPUStatsMap("cpu-msec"); }); #ifdef MALLOC_TRACE - addGetStat("memory-allocs", []{ return g_mtracer->getAllocs(string()); }); - addGetStat("memory-alloc-flux", []{ return g_mtracer->getAllocFlux(string()); }); - addGetStat("memory-allocated", []{ return g_mtracer->getTotAllocated(string()); }); + addGetStat("memory-allocs", [] { return g_mtracer->getAllocs(string()); }); + addGetStat("memory-alloc-flux", [] { return g_mtracer->getAllocFlux(string()); }); + addGetStat("memory-allocated", [] { return g_mtracer->getTotAllocated(string()); }); #endif addGetStat("dnssec-validations", &g_stats.dnssecValidations); addGetStat("dnssec-result-insecure", &g_stats.dnssecResults[vState::Insecure]); addGetStat("dnssec-result-secure", &g_stats.dnssecResults[vState::Secure]); addGetStat("dnssec-result-bogus", []() { - std::set const bogusStates = { vState::BogusNoValidDNSKEY, vState::BogusInvalidDenial, vState::BogusUnableToGetDSs, vState::BogusUnableToGetDNSKEYs, vState::BogusSelfSignedDS, vState::BogusNoRRSIG, vState::BogusNoValidRRSIG, vState::BogusMissingNegativeIndication, vState::BogusSignatureNotYetValid, vState::BogusSignatureExpired, vState::BogusUnsupportedDNSKEYAlgo, vState::BogusUnsupportedDSDigestType, vState::BogusNoZoneKeyBitSet, vState::BogusRevokedDNSKEY, vState::BogusInvalidDNSKEYProtocol }; + std::set const bogusStates = {vState::BogusNoValidDNSKEY, vState::BogusInvalidDenial, vState::BogusUnableToGetDSs, vState::BogusUnableToGetDNSKEYs, vState::BogusSelfSignedDS, vState::BogusNoRRSIG, vState::BogusNoValidRRSIG, vState::BogusMissingNegativeIndication, vState::BogusSignatureNotYetValid, vState::BogusSignatureExpired, vState::BogusUnsupportedDNSKEYAlgo, vState::BogusUnsupportedDSDigestType, vState::BogusNoZoneKeyBitSet, vState::BogusRevokedDNSKEY, vState::BogusInvalidDNSKEYProtocol}; uint64_t total = 0; for (const auto& state : bogusStates) { total += g_stats.dnssecResults[state]; @@ -1362,7 +1352,7 @@ static void registerAllStats1() if (::arg()["x-dnssec-names"].length() > 0) { addGetStat("x-dnssec-result-bogus", []() { - std::set const bogusStates = { vState::BogusNoValidDNSKEY, vState::BogusInvalidDenial, vState::BogusUnableToGetDSs, vState::BogusUnableToGetDNSKEYs, vState::BogusSelfSignedDS, vState::BogusNoRRSIG, vState::BogusNoValidRRSIG, vState::BogusMissingNegativeIndication, vState::BogusSignatureNotYetValid, vState::BogusSignatureExpired, vState::BogusUnsupportedDNSKEYAlgo, vState::BogusUnsupportedDSDigestType, vState::BogusNoZoneKeyBitSet, vState::BogusRevokedDNSKEY, vState::BogusInvalidDNSKEYProtocol }; + std::set const bogusStates = {vState::BogusNoValidDNSKEY, vState::BogusInvalidDenial, vState::BogusUnableToGetDSs, vState::BogusUnableToGetDNSKEYs, vState::BogusSelfSignedDS, vState::BogusNoRRSIG, vState::BogusNoValidRRSIG, vState::BogusMissingNegativeIndication, vState::BogusSignatureNotYetValid, vState::BogusSignatureExpired, vState::BogusUnsupportedDNSKEYAlgo, vState::BogusUnsupportedDSDigestType, vState::BogusNoZoneKeyBitSet, vState::BogusRevokedDNSKEY, vState::BogusInvalidDNSKEYProtocol}; uint64_t total = 0; for (const auto& state : bogusStates) { total += g_stats.xdnssecResults[state]; @@ -1403,15 +1393,17 @@ static void registerAllStats1() addGetStat("nod-lookups-dropped-oversize", &g_stats.nodLookupsDroppedOversize); - addGetStat("taskqueue-pushed", []() { return getTaskPushes(); }); - addGetStat("taskqueue-expired", []() { return getTaskExpired(); }); - addGetStat("taskqueue-size", []() { return getTaskSize(); }); + addGetStat("taskqueue-pushed", []() { return getTaskPushes(); }); + addGetStat("taskqueue-expired", []() { return getTaskExpired(); }); + addGetStat("taskqueue-size", []() { return getTaskSize(); }); + + addGetStat("dns64-prefix-answers", &g_stats.dns64prefixanswers); - addGetStat("dns64-prefix-answers", &g_stats.dns64prefixanswers); + addGetStat("almost-expired-pushed", []() { return getAlmostExpiredTasksPushed(); }); + addGetStat("almost-expired-run", []() { return getAlmostExpiredTasksRun(); }); + addGetStat("almost-expired-exceptions", []() { return getAlmostExpiredTaskExceptions(); }); - addGetStat("almost-expired-pushed", []() { return getAlmostExpiredTasksPushed(); }); - addGetStat("almost-expired-run", []() { return getAlmostExpiredTasksRun(); }); - addGetStat("almost-expired-exceptions", []() { return getAlmostExpiredTaskExceptions(); }); + addGetStat("idle-tcpout-connections", getCurrentIdleTCPConnections); /* make sure that the ECS stats are properly initialized */ SyncRes::clearECSStats(); @@ -1424,14 +1416,11 @@ static void registerAllStats1() addGetStat(name, &(SyncRes::s_ecsResponsesBySubnetSize6.at(idx))); } - addGetStat("cumul-answers", []() { + addGetStat("cumul-clientanswers", []() { return toStatsMap(g_stats.cumulativeAnswers.getName(), g_stats.cumulativeAnswers); }); - addGetStat("cumul-auth4answers", []() { - return toStatsMap(g_stats.cumulativeAuth4Answers.getName(), g_stats.cumulativeAuth4Answers); - }); - addGetStat("cumul-auth6answers", []() { - return toStatsMap(g_stats.cumulativeAuth6Answers.getName(), g_stats.cumulativeAuth6Answers); + addGetStat("cumul-authanswers", []() { + return toStatsMap(g_stats.cumulativeAuth4Answers.getName(), g_stats.cumulativeAuth4Answers, g_stats.cumulativeAuth6Answers); }); addGetStat("policy-hits", []() { return toRPZStatsMap("policy-hits", g_stats.policyHits); @@ -1441,28 +1430,30 @@ static void registerAllStats1() void registerAllStats() { static std::once_flag s_once; - std::call_once(s_once, []() { try { - registerAllStats1(); - } - catch (...) { - g_log << Logger::Critical << "Could not add stat entries" << endl; - exit(1); - } + std::call_once(s_once, []() { + try { + registerAllStats1(); + } + catch (...) { + g_log << Logger::Critical << "Could not add stat entries" << endl; + exit(1); + } }); } void doExitGeneric(bool nicely) { - g_log< >* pleaseGetQueryRing() +vector>* pleaseGetQueryRing() { - typedef pair query_t; - vector* ret = new vector(); - if(!t_queryring) + typedef pair query_t; + vector* ret = new vector(); + if (!t_queryring) return ret; ret->reserve(t_queryring->size()); - for(const query_t& q : *t_queryring) { + for (const query_t& q : *t_queryring) { ret->push_back(q); } return ret; } -vector >* pleaseGetServfailQueryRing() +vector>* pleaseGetServfailQueryRing() { - typedef pair query_t; + typedef pair query_t; vector* ret = new vector(); - if(!t_servfailqueryring) + if (!t_servfailqueryring) return ret; ret->reserve(t_servfailqueryring->size()); - for(const query_t& q : *t_servfailqueryring) { + for (const query_t& q : *t_servfailqueryring) { ret->push_back(q); } return ret; } -vector >* pleaseGetBogusQueryRing() +vector>* pleaseGetBogusQueryRing() { - typedef pair query_t; + typedef pair query_t; vector* ret = new vector(); - if(!t_bogusqueryring) + if (!t_bogusqueryring) return ret; ret->reserve(t_bogusqueryring->size()); - for(const query_t& q : *t_bogusqueryring) { + for (const query_t& q : *t_bogusqueryring) { ret->push_back(q); } return ret; } - - typedef boost::function*()> pleaseremotefunc_t; -typedef boost::function >*()> pleasequeryfunc_t; +typedef boost::function>*()> pleasequeryfunc_t; vector* pleaseGetRemotes() { vector* ret = new vector(); - if(!t_remotes) + if (!t_remotes) return ret; ret->reserve(t_remotes->size()); - for(const ComboAddress& ca : *t_remotes) { + for (const ComboAddress& ca : *t_remotes) { ret->push_back(ca); } return ret; @@ -1536,10 +1525,10 @@ vector* pleaseGetRemotes() vector* pleaseGetServfailRemotes() { vector* ret = new vector(); - if(!t_servfailremotes) + if (!t_servfailremotes) return ret; ret->reserve(t_servfailremotes->size()); - for(const ComboAddress& ca : *t_servfailremotes) { + for (const ComboAddress& ca : *t_servfailremotes) { ret->push_back(ca); } return ret; @@ -1548,10 +1537,10 @@ vector* pleaseGetServfailRemotes() vector* pleaseGetBogusRemotes() { vector* ret = new vector(); - if(!t_bogusremotes) + if (!t_bogusremotes) return ret; ret->reserve(t_bogusremotes->size()); - for(const ComboAddress& ca : *t_bogusremotes) { + for (const ComboAddress& ca : *t_bogusremotes) { ret->push_back(ca); } return ret; @@ -1560,10 +1549,10 @@ vector* pleaseGetBogusRemotes() vector* pleaseGetLargeAnswerRemotes() { vector* ret = new vector(); - if(!t_largeanswerremotes) + if (!t_largeanswerremotes) return ret; ret->reserve(t_largeanswerremotes->size()); - for(const ComboAddress& ca : *t_largeanswerremotes) { + for (const ComboAddress& ca : *t_largeanswerremotes) { ret->push_back(ca); } return ret; @@ -1572,10 +1561,10 @@ vector* pleaseGetLargeAnswerRemotes() vector* pleaseGetTimeouts() { vector* ret = new vector(); - if(!t_timeouts) + if (!t_timeouts) return ret; ret->reserve(t_timeouts->size()); - for(const ComboAddress& ca : *t_timeouts) { + for (const ComboAddress& ca : *t_timeouts) { ret->push_back(ca); } return ret; @@ -1586,30 +1575,31 @@ static string doGenericTopRemotes(pleaseremotefunc_t func) typedef map counts_t; counts_t counts; - vector remotes=broadcastAccFunction >(func); - - unsigned int total=0; - for(const ComboAddress& ca : remotes) { + vector remotes = broadcastAccFunction>(func); + + unsigned int total = 0; + for (const ComboAddress& ca : remotes) { total++; counts[ca]++; } - + typedef std::multimap rcounts_t; rcounts_t rcounts; - - for(counts_t::const_iterator i=counts.begin(); i != counts.end(); ++i) - rcounts.insert(make_pair(-i->second, i->first)); + + for (auto&& c : counts) + rcounts.emplace(-c.second, c.first); ostringstream ret; - ret<<"Over last "<first/total) % i->second.toString(); - accounted+= -i->first; + int limit = 0, accounted = 0; + if (total) { + for (rcounts_t::const_iterator i = rcounts.begin(); i != rcounts.end() && limit < 20; ++i, ++limit) { + ret << fmt % (-100.0 * i->first / total) % i->second.toString(); + accounted += -i->first; } - ret<< '\n' << fmt % (100.0*(total-accounted)/total) % "rest"; + ret << '\n' + << fmt % (100.0 * (total - accounted) / total) % "rest"; } return ret.str(); } @@ -1617,29 +1607,31 @@ static string doGenericTopRemotes(pleaseremotefunc_t func) // XXX DNSName Pain - this function should benefit from native DNSName methods DNSName getRegisteredName(const DNSName& dom) { - auto parts=dom.getRawLabels(); - if(parts.size()<=2) + auto parts = dom.getRawLabels(); + if (parts.size() <= 2) return dom; reverse(parts.begin(), parts.end()); - for(string& str : parts) { str=toLower(str); }; + for (string& str : parts) { + str = toLower(str); + }; - // uk co migweb + // uk co migweb string last; - while(!parts.empty()) { - if(parts.size()==1 || binary_search(g_pubs.begin(), g_pubs.end(), parts)) { - - string ret=last; - if(!ret.empty()) - ret+="."; - - for(auto p = parts.crbegin(); p != parts.crend(); ++p) { - ret+=(*p)+"."; + while (!parts.empty()) { + if (parts.size() == 1 || binary_search(g_pubs.begin(), g_pubs.end(), parts)) { + + string ret = last; + if (!ret.empty()) + ret += "."; + + for (auto p = parts.crbegin(); p != parts.crend(); ++p) { + ret += (*p) + "."; } return DNSName(ret); } - last=parts[parts.size()-1]; - parts.resize(parts.size()-1); + last = parts[parts.size() - 1]; + parts.resize(parts.size() - 1); } return DNSName("??"); } @@ -1649,38 +1641,38 @@ static DNSName nopFilter(const DNSName& name) return name; } -static string doGenericTopQueries(pleasequeryfunc_t func, boost::function filter=nopFilter) +static string doGenericTopQueries(pleasequeryfunc_t func, boost::function filter = nopFilter) { - typedef pair query_t; + typedef pair query_t; typedef map counts_t; counts_t counts; - vector queries=broadcastAccFunction >(func); - - unsigned int total=0; - for(const query_t& q : queries) { + vector queries = broadcastAccFunction>(func); + + unsigned int total = 0; + for (const query_t& q : queries) { total++; - counts[make_pair(filter(q.first),q.second)]++; + counts[pair(filter(q.first), q.second)]++; } typedef std::multimap rcounts_t; rcounts_t rcounts; - - for(counts_t::const_iterator i=counts.begin(); i != counts.end(); ++i) - rcounts.insert(make_pair(-i->second, i->first)); + + for (auto&& c : counts) + rcounts.emplace(-c.second, c.first); ostringstream ret; - ret<<"Over last "<first/total) % (i->second.first.toLogString()+"|"+DNSRecordContent::NumberToType(i->second.second)); - accounted+= -i->first; + int limit = 0, accounted = 0; + if (total) { + for (rcounts_t::const_iterator i = rcounts.begin(); i != rcounts.end() && limit < 20; ++i, ++limit) { + ret << fmt % (-100.0 * i->first / total) % (i->second.first.toLogString() + "|" + DNSRecordContent::NumberToType(i->second.second)); + accounted += -i->first; } - ret<< '\n' << fmt % (100.0*(total-accounted)/total) % "rest"; + ret << '\n' + << fmt % (100.0 * (total - accounted) / total) % "rest"; } - return ret.str(); } @@ -1689,18 +1681,21 @@ static string* nopFunction() return new string("pong\n"); } -static string getDontThrottleNames() { +static string getDontThrottleNames() +{ auto dtn = g_dontThrottleNames.getLocal(); return dtn->toString() + "\n"; } -static string getDontThrottleNetmasks() { +static string getDontThrottleNetmasks() +{ auto dtn = g_dontThrottleNetmasks.getLocal(); return dtn->toString() + "\n"; } -template -static string addDontThrottleNames(T begin, T end) { +template +static string addDontThrottleNames(T begin, T end) +{ if (begin == end) { return "No names specified, keeping existing list\n"; } @@ -1710,8 +1705,8 @@ static string addDontThrottleNames(T begin, T end) { auto d = DNSName(*begin); toAdd.push_back(d); } - catch(const std::exception &e) { - return "Problem parsing '" + *begin + "': "+ e.what() + ", nothing added\n"; + catch (const std::exception& e) { + return "Problem parsing '" + *begin + "': " + e.what() + ", nothing added\n"; } begin++; } @@ -1719,7 +1714,7 @@ static string addDontThrottleNames(T begin, T end) { string ret = "Added"; auto dnt = g_dontThrottleNames.getCopy(); bool first = true; - for (auto const &d : toAdd) { + for (auto const& d : toAdd) { if (!first) { ret += ","; } @@ -1731,12 +1726,13 @@ static string addDontThrottleNames(T begin, T end) { g_dontThrottleNames.setState(std::move(dnt)); ret += " to the list of nameservers that may not be throttled"; - g_log< -static string addDontThrottleNetmasks(T begin, T end) { +template +static string addDontThrottleNetmasks(T begin, T end) +{ if (begin == end) { return "No netmasks specified, keeping existing list\n"; } @@ -1746,11 +1742,11 @@ static string addDontThrottleNetmasks(T begin, T end) { auto n = Netmask(*begin); toAdd.push_back(n); } - catch(const std::exception &e) { - return "Problem parsing '" + *begin + "': "+ e.what() + ", nothing added\n"; + catch (const std::exception& e) { + return "Problem parsing '" + *begin + "': " + e.what() + ", nothing added\n"; } - catch(const PDNSException &e) { - return "Problem parsing '" + *begin + "': "+ e.reason + ", nothing added\n"; + catch (const PDNSException& e) { + return "Problem parsing '" + *begin + "': " + e.reason + ", nothing added\n"; } begin++; } @@ -1758,7 +1754,7 @@ static string addDontThrottleNetmasks(T begin, T end) { string ret = "Added"; auto dnt = g_dontThrottleNetmasks.getCopy(); bool first = true; - for (auto const &t : toAdd) { + for (auto const& t : toAdd) { if (!first) { ret += ","; } @@ -1770,20 +1766,21 @@ static string addDontThrottleNetmasks(T begin, T end) { g_dontThrottleNetmasks.setState(std::move(dnt)); ret += " to the list of nameserver netmasks that may not be throttled"; - g_log< -static string clearDontThrottleNames(T begin, T end) { - if(begin == end) +template +static string clearDontThrottleNames(T begin, T end) +{ + if (begin == end) return "No names specified, doing nothing.\n"; - if (begin + 1 == end && *begin == "*"){ + if (begin + 1 == end && *begin == "*") { SuffixMatchNode smn; g_dontThrottleNames.setState(std::move(smn)); string ret = "Cleared list of nameserver names that may not be throttled"; - g_log< -static string clearDontThrottleNetmasks(T begin, T end) { - if(begin == end) +template +static string clearDontThrottleNetmasks(T begin, T end) +{ + if (begin == end) return "No netmasks specified, doing nothing.\n"; - if (begin + 1 == end && *begin == "*"){ + if (begin + 1 == end && *begin == "*") { auto nmg = g_dontThrottleNetmasks.getCopy(); nmg.clear(); g_dontThrottleNetmasks.setState(std::move(nmg)); string ret = "Cleared list of nameserver addresses that may not be throttled"; - g_log< +static string setEventTracing(T begin, T end) +{ + if (begin == end) { + return "No event trace enabled value specified\n"; + } + try { + SyncRes::s_event_trace_enabled = pdns_stou(*begin); + return "New event trace enabled value: " + std::to_string(SyncRes::s_event_trace_enabled) + "\n"; + } + catch (const std::exception& e) { + return "Error parsing the new event trace enabled value: " + std::string(e.what()) + "\n"; + } +} RecursorControlChannel::Answer RecursorControlParser::getAnswer(int s, const string& question, RecursorControlParser::func_t** command) { - *command=nop; + *command = nop; vector words; stringtok(words, question); - if(words.empty()) + if (words.empty()) return {1, "invalid command\n"}; - string cmd=toLower(words[0]); - vector::const_iterator begin=words.begin()+1, end=words.end(); + string cmd = toLower(words[0]); + vector::const_iterator begin = words.begin() + 1, end = words.end(); // should probably have a smart dispatcher here, like auth has - if(cmd=="help") + if (cmd == "help") return {0, -"add-dont-throttle-names [N...] add names that are not allowed to be throttled\n" -"add-dont-throttle-netmasks [N...]\n" -" add netmasks that are not allowed to be throttled\n" -"add-nta DOMAIN [REASON] add a Negative Trust Anchor for DOMAIN with the comment REASON\n" -"add-ta DOMAIN DSRECORD add a Trust Anchor for DOMAIN with data DSRECORD\n" -"current-queries show currently active queries\n" -"clear-dont-throttle-names [N...] remove names that are not allowed to be throttled. If N is '*', remove all\n" -"clear-dont-throttle-netmasks [N...]\n" -" remove netmasks that are not allowed to be throttled. If N is '*', remove all\n" -"clear-nta [DOMAIN]... Clear the Negative Trust Anchor for DOMAINs, if no DOMAIN is specified, remove all\n" -"clear-ta [DOMAIN]... Clear the Trust Anchor for DOMAINs\n" -"dump-cache dump cache contents to the named file\n" -"dump-edns [status] dump EDNS status to the named file\n" -"dump-failedservers dump the failed servers to the named file\n" -"dump-non-resolving dump non-resolving nameservers addresses to the named file\n" -"dump-nsspeeds dump nsspeeds statistics to the named file\n" -"dump-rpz dump the content of a RPZ zone to the named file\n" -"dump-throttlemap dump the contents of the throttle map to the named file\n" -"get [key1] [key2] .. get specific statistics\n" -"get-all get all statistics\n" -"get-dont-throttle-names get the list of names that are not allowed to be throttled\n" -"get-dont-throttle-netmasks get the list of netmasks that are not allowed to be throttled\n" -"get-ntas get all configured Negative Trust Anchors\n" -"get-tas get all configured Trust Anchors\n" -"get-parameter [key1] [key2] .. get configuration parameters\n" -"get-qtypelist get QType statistics\n" -" notice: queries from cache aren't being counted yet\n" -"help get this list\n" -"ping check that all threads are alive\n" -"quit stop the recursor daemon\n" -"quit-nicely stop the recursor daemon nicely\n" -"reload-acls reload ACLS\n" -"reload-lua-script [filename] (re)load Lua script\n" -"reload-lua-config [filename] (re)load Lua configuration file\n" -"reload-zones reload all auth and forward zones\n" -"set-ecs-minimum-ttl value set ecs-minimum-ttl-override\n" -"set-max-cache-entries value set new maximum cache size\n" -"set-max-packetcache-entries val set new maximum packet cache size\n" -"set-minimum-ttl value set minimum-ttl-override\n" -"set-carbon-server set a carbon server for telemetry\n" -"set-dnssec-log-bogus SETTING enable (SETTING=yes) or disable (SETTING=no) logging of DNSSEC validation failures\n" -"trace-regex [regex] emit resolution trace for matching queries (empty regex to clear trace)\n" -"top-largeanswer-remotes show top remotes receiving large answers\n" -"top-queries show top queries\n" -"top-pub-queries show top queries grouped by public suffix list\n" -"top-remotes show top remotes\n" -"top-timeouts show top downstream timeouts\n" -"top-servfail-queries show top queries receiving servfail answers\n" -"top-bogus-queries show top queries validating as bogus\n" -"top-pub-servfail-queries show top queries receiving servfail answers grouped by public suffix list\n" -"top-pub-bogus-queries show top queries validating as bogus grouped by public suffix list\n" -"top-servfail-remotes show top remotes receiving servfail answers\n" -"top-bogus-remotes show top remotes receiving bogus answers\n" -"unload-lua-script unload Lua script\n" -"version return Recursor version number\n" -"wipe-cache domain0 [domain1] .. wipe domain data from cache\n" -"wipe-cache-typed type domain0 [domain1] .. wipe domain data with qtype from cache\n"}; + "add-dont-throttle-names [N...] add names that are not allowed to be throttled\n" + "add-dont-throttle-netmasks [N...]\n" + " add netmasks that are not allowed to be throttled\n" + "add-nta DOMAIN [REASON] add a Negative Trust Anchor for DOMAIN with the comment REASON\n" + "add-ta DOMAIN DSRECORD add a Trust Anchor for DOMAIN with data DSRECORD\n" + "current-queries show currently active queries\n" + "clear-dont-throttle-names [N...] remove names that are not allowed to be throttled. If N is '*', remove all\n" + "clear-dont-throttle-netmasks [N...]\n" + " remove netmasks that are not allowed to be throttled. If N is '*', remove all\n" + "clear-nta [DOMAIN]... Clear the Negative Trust Anchor for DOMAINs, if no DOMAIN is specified, remove all\n" + "clear-ta [DOMAIN]... Clear the Trust Anchor for DOMAINs\n" + "dump-cache dump cache contents to the named file\n" + "dump-edns [status] dump EDNS status to the named file\n" + "dump-failedservers dump the failed servers to the named file\n" + "dump-non-resolving dump non-resolving nameservers addresses to the named file\n" + "dump-nsspeeds dump nsspeeds statistics to the named file\n" + "dump-rpz dump the content of a RPZ zone to the named file\n" + "dump-throttlemap dump the contents of the throttle map to the named file\n" + "get [key1] [key2] .. get specific statistics\n" + "get-all get all statistics\n" + "get-dont-throttle-names get the list of names that are not allowed to be throttled\n" + "get-dont-throttle-netmasks get the list of netmasks that are not allowed to be throttled\n" + "get-ntas get all configured Negative Trust Anchors\n" + "get-tas get all configured Trust Anchors\n" + "get-parameter [key1] [key2] .. get configuration parameters\n" + "get-qtypelist get QType statistics\n" + " notice: queries from cache aren't being counted yet\n" + "hash-password [work-factor] ask for a password then return the hashed version\n" + "help get this list\n" + "ping check that all threads are alive\n" + "quit stop the recursor daemon\n" + "quit-nicely stop the recursor daemon nicely\n" + "reload-acls reload ACLS\n" + "reload-lua-script [filename] (re)load Lua script\n" + "reload-lua-config [filename] (re)load Lua configuration file\n" + "reload-zones reload all auth and forward zones\n" + "set-ecs-minimum-ttl value set ecs-minimum-ttl-override\n" + "set-max-cache-entries value set new maximum cache size\n" + "set-max-packetcache-entries val set new maximum packet cache size\n" + "set-minimum-ttl value set minimum-ttl-override\n" + "set-carbon-server set a carbon server for telemetry\n" + "set-dnssec-log-bogus SETTING enable (SETTING=yes) or disable (SETTING=no) logging of DNSSEC validation failures\n" + "set-event-trace-enabled SETTING set logging of event trace messages, 0 = disabled, 1 = protobuf, 2 = log file, 3 = both\n" + "trace-regex [regex] emit resolution trace for matching queries (empty regex to clear trace)\n" + "top-largeanswer-remotes show top remotes receiving large answers\n" + "top-queries show top queries\n" + "top-pub-queries show top queries grouped by public suffix list\n" + "top-remotes show top remotes\n" + "top-timeouts show top downstream timeouts\n" + "top-servfail-queries show top queries receiving servfail answers\n" + "top-bogus-queries show top queries validating as bogus\n" + "top-pub-servfail-queries show top queries receiving servfail answers grouped by public suffix list\n" + "top-pub-bogus-queries show top queries validating as bogus grouped by public suffix list\n" + "top-servfail-remotes show top remotes receiving servfail answers\n" + "top-bogus-remotes show top remotes receiving bogus answers\n" + "unload-lua-script unload Lua script\n" + "version return Recursor version number\n" + "wipe-cache domain0 [domain1] .. wipe domain data from cache\n" + "wipe-cache-typed type domain0 [domain1] .. wipe domain data with qtype from cache\n"}; if (cmd == "get-all") { return {0, getAllStats()}; @@ -1956,14 +1970,14 @@ RecursorControlChannel::Answer RecursorControlParser::getAnswer(int s, const str return {0, doGetParameter(begin, end)}; } if (cmd == "quit") { - *command=&doExit; + *command = &doExit; return {0, "bye\n"}; } if (cmd == "version") { - return {0, getPDNSVersion()+"\n"}; + return {0, getPDNSVersion() + "\n"}; } if (cmd == "quit-nicely") { - *command=&doExitNicely; + *command = &doExitNicely; return {0, "bye nicely\n"}; } if (cmd == "dump-cache") { @@ -1991,7 +2005,13 @@ RecursorControlChannel::Answer RecursorControlParser::getAnswer(int s, const str return {0, doWipeCache(begin, end, 0xffff)}; } if (cmd == "wipe-cache-typed") { + if (begin == end) { + return {1, "Need a qtype\n"}; + } uint16_t qtype = QType::chartocode(begin->c_str()); + if (qtype == 0) { + return {1, "Unknown qtype " + *begin + "\n"}; + } ++begin; return {0, doWipeCache(begin, end, qtype)}; } @@ -2006,14 +2026,14 @@ RecursorControlChannel::Answer RecursorControlParser::getAnswer(int s, const str luaConfigDelayedThreads delayedLuaThreads; loadRecursorLuaConfig(::arg()["lua-config-file"], delayedLuaThreads); startLuaConfigDelayedThreads(delayedLuaThreads, g_luaconfs.getCopy().generation); - g_log< #include + #include "pdnsexception.hh" #include "arguments.hh" - +#include "credentials.hh" #include "namespaces.hh" +#include "rec_channel.hh" -ArgvMap &arg() +ArgvMap& arg() { static ArgvMap arg; return arg; @@ -38,45 +40,47 @@ ArgvMap &arg() static void initArguments(int argc, char** argv) { - arg().set("config-dir","Location of configuration directory (recursor.conf)")=SYSCONFDIR; - - arg().set("socket-dir",string("Where the controlsocket will live, ")+LOCALSTATEDIR+"/pdns-recursor when unset and not chrooted" )=""; - arg().set("chroot","switch to chroot jail")=""; - arg().set("process","When controlling multiple recursors, the target process number")=""; - arg().set("timeout", "Number of seconds to wait for the recursor to respond")="5"; - arg().set("config-name","Name of this virtual configuration - will rename the binary image")=""; - arg().setCmd("help","Provide this helpful message"); - arg().setCmd("version","Show the version of this program"); - - arg().laxParse(argc,argv); - if(arg().mustDo("help") || arg().getCommands().empty()) { - cout<<"syntax: rec_control [options] command, options as below: "<& commands = arg().getCommands(); + + if (commands.size() >= 1 && commands.at(0) == "hash-password") { + uint64_t workFactor = CredentialsHolder::s_defaultWorkFactor; + if (commands.size() > 1) { + try { + workFactor = pdns_stou(commands.at(1)); + } + catch (const std::exception& e) { + cerr << "Unable to parse the supplied work factor: " << e.what() << endl; + return EXIT_FAILURE; + } + } + + auto password = CredentialsHolder::readFromTerminal(); + + try { + cout << hashPassword(password.getString(), workFactor, CredentialsHolder::s_defaultParallelFactor, CredentialsHolder::s_defaultBlockSize) << endl; + return EXIT_SUCCESS; + } + catch (const std::exception& e) { + cerr << "Error while hashing the supplied password: " << e.what() << endl; + return EXIT_FAILURE; + } + } - const vector&commands=arg().getCommands(); string command; int fd = -1; unsigned int i = 0; @@ -124,21 +149,24 @@ int main(int argc, char** argv) if (i + 2 < commands.size()) { ++i; command += " " + commands[i]; // add rpzname and continue with filename - } else { + } + else { throw PDNSException("Command needs a zone and file argument"); } } ++i; if (commands[i] == "-") { fd = STDOUT_FILENO; - } else { + } + else { fd = open(commands[i].c_str(), O_CREAT | O_EXCL | O_WRONLY, 0660); } if (fd == -1) { int err = errno; throw PDNSException("Error opening dump file for writing: " + stringerror(err)); } - } else { + } + else { throw PDNSException("Command needs a file argument"); } } @@ -146,19 +174,21 @@ int main(int argc, char** argv) } auto timeout = arg().asNum("timeout"); - rccS.send({0, command}, nullptr, timeout, fd); + RecursorControlChannel rccS; + rccS.connect(arg()["socket-dir"], sockname); + rccS.send(rccS.d_fd, {0, command}, timeout, fd); - auto receive = rccS.recv(0, timeout); + auto receive = rccS.recv(rccS.d_fd, timeout); if (receive.d_ret != 0) { cerr << receive.d_str; - } else { + } + else { cout << receive.d_str; } return receive.d_ret; } - catch(PDNSException& ae) { - cerr<<"Fatal: "<&parts, pid_t ppid) @@ -583,8 +583,8 @@ int main(int argc, char **argv) if(::arg().mustDo("control-console")) dl=make_unique(); else - dl=std::unique_ptr(new DynListener(s_programname)); - + dl = std::make_unique(s_programname); + writePid(); } DynListener::registerFunc("SHOW",&DLShowHandler, "show a specific statistic or * to get a list", ""); diff --git a/pdns/recpacketcache.cc b/pdns/recpacketcache.cc index d22447f8364f..dc36d898e93f 100644 --- a/pdns/recpacketcache.cc +++ b/pdns/recpacketcache.cc @@ -19,21 +19,21 @@ unsigned int RecursorPacketCache::s_refresh_ttlperc{0}; int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype, bool subtree) { - int count=0; + int count = 0; auto& idx = d_packetCache.get(); - for(auto iter = idx.lower_bound(name); iter != idx.end(); ) { - if(subtree) { - if(!iter->d_name.isPartOf(name)) { // this is case insensitive - break; + for (auto iter = idx.lower_bound(name); iter != idx.end();) { + if (subtree) { + if (!iter->d_name.isPartOf(name)) { // this is case insensitive + break; } } else { - if(iter->d_name != name) - break; + if (iter->d_name != name) + break; } - if(qtype==0xffff || iter->d_type == qtype) { - iter=idx.erase(iter); + if (qtype == 0xffff || iter->d_type == qtype) { + iter = idx.erase(iter); count++; } else @@ -49,13 +49,13 @@ bool RecursorPacketCache::qrMatch(const packetCache_t::index::type::ite return false; } - static const std::unordered_set optionsToSkip{ EDNSOptionCode::COOKIE, EDNSOptionCode::ECS }; + static const std::unordered_set optionsToSkip{EDNSOptionCode::COOKIE, EDNSOptionCode::ECS}; return queryMatches(iter->d_query, queryPacket, qname, optionsToSkip); } bool RecursorPacketCache::checkResponseMatches(std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, OptPBData* pbdata) { - for(auto iter = range.first ; iter != range.second ; ++iter) { + for (auto iter = range.first; iter != range.second; ++iter) { // the possibility is VERY real that we get hits that are not right - birthday paradox if (!qrMatch(iter, queryPacket, qname, qtype, qclass)) { continue; @@ -88,7 +88,8 @@ bool RecursorPacketCache::checkResponseMatches(std::paird_pbdata) { *pbdata = iter->d_pbdata; - } else { + } + else { *pbdata = boost::none; } } @@ -121,14 +122,16 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& return getResponsePacket(tag, queryPacket, qname, qtype, qclass, now, responsePacket, age, &valState, qhash, nullptr, false); } +static const std::unordered_set s_skipOptions = {EDNSOptionCode::ECS, EDNSOptionCode::COOKIE}; + bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, OptPBData* pbdata, bool tcp) { - *qhash = canHashPacket(queryPacket, true); + *qhash = canHashPacket(queryPacket, s_skipOptions); const auto& idx = d_packetCache.get(); auto range = idx.equal_range(tie(tag, *qhash, tcp)); - if(range.first == range.second) { + if (range.first == range.second) { d_misses++; return false; } @@ -137,13 +140,13 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& } bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, DNSName& qname, uint16_t* qtype, uint16_t* qclass, time_t now, - std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, OptPBData *pbdata, bool tcp) + std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, OptPBData* pbdata, bool tcp) { - *qhash = canHashPacket(queryPacket, true); + *qhash = canHashPacket(queryPacket, s_skipOptions); const auto& idx = d_packetCache.get(); auto range = idx.equal_range(tie(tag, *qhash, tcp)); - if(range.first == range.second) { + if (range.first == range.second) { d_misses++; return false; } @@ -153,15 +156,14 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& return checkResponseMatches(range, queryPacket, qname, *qtype, *qclass, now, responsePacket, age, valState, pbdata); } - void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, std::string&& query, const DNSName& qname, uint16_t qtype, uint16_t qclass, std::string&& responsePacket, time_t now, uint32_t ttl, const vState& valState, OptPBData&& pbdata, bool tcp) { auto& idx = d_packetCache.get(); auto range = idx.equal_range(tie(tag, qhash, tcp)); auto iter = range.first; - for( ; iter != range.second ; ++iter) { - if (iter->d_type != qtype || iter->d_class != qclass || iter->d_name != qname ) { + for (; iter != range.second; ++iter) { + if (iter->d_type != qtype || iter->d_class != qclass || iter->d_name != qname) { continue; } @@ -179,7 +181,7 @@ void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, break; } - if(iter == range.second) { // nothing to refresh + if (iter == range.second) { // nothing to refresh struct Entry e(qname, std::move(responsePacket), std::move(query), tcp); e.d_qhash = qhash; e.d_type = qtype; @@ -203,8 +205,8 @@ uint64_t RecursorPacketCache::size() uint64_t RecursorPacketCache::bytes() { - uint64_t sum=0; - for(const auto& e : d_packetCache) { + uint64_t sum = 0; + for (const auto& e : d_packetCache) { sum += sizeof(e) + e.d_packet.length() + 4; } return sum; @@ -217,7 +219,7 @@ void RecursorPacketCache::doPruneTo(size_t maxCached) uint64_t RecursorPacketCache::doDump(int fd) { - auto fp = std::unique_ptr(fdopen(dup(fd), "w"), fclose); + auto fp = std::unique_ptr(fdopen(dup(fd), "w"), fclose); if (!fp) { // dup probably failed return 0; } @@ -233,7 +235,7 @@ uint64_t RecursorPacketCache::doDump(int fd) try { fprintf(fp.get(), "%s %" PRId64 " %s ; tag %d %s\n", i.d_name.toString().c_str(), static_cast(i.d_ttd - now), DNSRecordContent::NumberToType(i.d_type).c_str(), i.d_tag, i.d_tcp ? "tcp" : "udp"); } - catch(...) { + catch (...) { fprintf(fp.get(), "; error printing '%s'\n", i.d_name.empty() ? "EMPTY" : i.d_name.toString().c_str()); } } diff --git a/pdns/recpacketcache.hh b/pdns/recpacketcache.hh index 7ef8a99e9d3e..a88fc558408a 100644 --- a/pdns/recpacketcache.hh +++ b/pdns/recpacketcache.hh @@ -46,12 +46,13 @@ using namespace ::boost::multi_index; you can use a query as a key too. But query and answer must compare as identical! This precludes doing anything smart with EDNS directly from the packet */ -class RecursorPacketCache: public PacketCache +class RecursorPacketCache : public PacketCache { public: static unsigned int s_refresh_ttlperc; - struct PBData { + struct PBData + { std::string d_message; std::string d_response; bool d_tagged; @@ -62,24 +63,29 @@ public: bool getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, std::string* responsePacket, uint32_t* age, uint32_t* qhash); bool getResponsePacket(unsigned int tag, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, uint32_t* qhash); bool getResponsePacket(unsigned int tag, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, OptPBData* pbdata, bool tcp); - bool getResponsePacket(unsigned int tag, const std::string& queryPacket, DNSName& qname, uint16_t *qtype, uint16_t* qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, OptPBData* pbdata, bool tcp); - + bool getResponsePacket(unsigned int tag, const std::string& queryPacket, DNSName& qname, uint16_t* qtype, uint16_t* qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, OptPBData* pbdata, bool tcp); + void insertResponsePacket(unsigned int tag, uint32_t qhash, std::string&& query, const DNSName& qname, uint16_t qtype, uint16_t qclass, std::string&& responsePacket, time_t now, uint32_t ttl, const vState& valState, OptPBData&& pbdata, bool tcp); - void doPruneTo(size_t maxSize=250000); + void doPruneTo(size_t maxSize = 250000); uint64_t doDump(int fd); - int doWipePacketCache(const DNSName& name, uint16_t qtype=0xffff, bool subtree=false); - + int doWipePacketCache(const DNSName& name, uint16_t qtype = 0xffff, bool subtree = false); + void prune(); uint64_t d_hits, d_misses; uint64_t size(); uint64_t bytes(); private: - struct HashTag {}; - struct NameTag {}; - struct Entry + struct HashTag + { + }; + struct NameTag { - Entry(const DNSName& qname, std::string&& packet, std::string&& query, bool tcp): d_name(qname), d_packet(std::move(packet)), d_query(std::move(query)), d_tcp(tcp) + }; + struct Entry + { + Entry(const DNSName& qname, std::string&& packet, std::string&& query, bool tcp) : + d_name(qname), d_packet(std::move(packet)), d_query(std::move(query)), d_tcp(tcp) { } @@ -88,14 +94,14 @@ private: mutable std::string d_query; mutable OptPBData d_pbdata; mutable time_t d_ttd; - mutable time_t d_creation; // so we can 'age' our packets + mutable time_t d_creation; // so we can 'age' our packets uint32_t d_qhash; uint32_t d_tag; uint16_t d_type; uint16_t d_class; mutable vState d_vstate; mutable bool d_submitted{false}; // whether this entry has been queued for refetch - bool d_tcp; // whether this entry was created from a TCP query + bool d_tcp; // whether this entry was created from a TCP query inline bool operator<(const struct Entry& rhs) const; time_t getTTD() const @@ -103,26 +109,26 @@ private: return d_ttd; } - uint32_t getOrigTTL() const { + uint32_t getOrigTTL() const + { return d_ttd - d_creation; } }; - struct SequencedTag{}; + struct SequencedTag + { + }; typedef multi_index_container< Entry, - indexed_by < + indexed_by< hashed_non_unique, composite_key, member, - member - > - >, + member>>, sequenced>, - ordered_non_unique, member, CanonDNSNameCompare> - > - > packetCache_t; + ordered_non_unique, member, CanonDNSNameCompare>>> + packetCache_t; packetCache_t d_packetCache; diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc index a630a2e49612..b20be391b180 100644 --- a/pdns/recursor_cache.cc +++ b/pdns/recursor_cache.cc @@ -15,7 +15,8 @@ #include "cachecleaner.hh" #include "rec-taskqueue.hh" -MemRecursorCache::MemRecursorCache(size_t mapsCount) : d_maps(mapsCount) +MemRecursorCache::MemRecursorCache(size_t mapsCount) : + d_maps(mapsCount) { } @@ -28,7 +29,7 @@ size_t MemRecursorCache::size() const return count; } -pair MemRecursorCache::stats() +pair MemRecursorCache::stats() { uint64_t c = 0, a = 0; for (auto& mc : d_maps) { @@ -36,7 +37,7 @@ pair MemRecursorCache::stats() c += content->d_contended_count; a += content->d_acquired_count; } - return pair(c, a); + return pair(c, a); } size_t MemRecursorCache::ecsIndexSize() @@ -112,7 +113,7 @@ time_t MemRecursorCache::handleHit(MapCombo::LockedContent& content, MemRecursor if (res) { res->reserve(res->size() + entry->d_records.size()); - for(const auto& k : entry->d_records) { + for (const auto& k : entry->d_records) { DNSRecord dr; dr.d_name = qname; dr.d_type = entry->d_qtype; @@ -147,7 +148,7 @@ time_t MemRecursorCache::handleHit(MapCombo::LockedContent& content, MemRecursor return ttd; } -MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSIndex(MapCombo::LockedContent& map, time_t now, const DNSName &qname, const QType qtype, bool requireAuth, const ComboAddress& who) +MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSIndex(MapCombo::LockedContent& map, time_t now, const DNSName& qname, const QType qtype, bool requireAuth, const ComboAddress& who) { // MUTEX SHOULD BE ACQUIRED (as indicated by the reference to the content which is protected by a lock) auto ecsIndexKey = tie(qname, qtype); @@ -209,7 +210,7 @@ MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSInde return map.d_map.end(); } -MemRecursorCache::Entries MemRecursorCache::getEntries(MapCombo::LockedContent& map, const DNSName &qname, const QType qt, const OptTag& rtag ) +MemRecursorCache::Entries MemRecursorCache::getEntries(MapCombo::LockedContent& map, const DNSName& qname, const QType qt, const OptTag& rtag) { // MUTEX SHOULD BE ACQUIRED if (!map.d_cachecachevalid || map.d_cachedqname != qname || map.d_cachedrtag != rtag) { @@ -222,7 +223,6 @@ MemRecursorCache::Entries MemRecursorCache::getEntries(MapCombo::LockedContent& return map.d_cachecache; } - bool MemRecursorCache::entryMatches(MemRecursorCache::OrderedTagIterator_t& entry, const QType qt, bool requireAuth, const ComboAddress& who) { // This code assumes that if a routing tag is present, it matches @@ -230,8 +230,7 @@ bool MemRecursorCache::entryMatches(MemRecursorCache::OrderedTagIterator_t& entr if (requireAuth && !entry->d_auth) return false; - bool match = (entry->d_qtype == qt || qt == QType::ANY || - (qt == QType::ADDR && (entry->d_qtype == QType::A || entry->d_qtype == QType::AAAA))) + bool match = (entry->d_qtype == qt || qt == QType::ANY || (qt == QType::ADDR && (entry->d_qtype == QType::A || entry->d_qtype == QType::AAAA))) && (entry->d_netmask.empty() || entry->d_netmask.match(who)); return match; } @@ -246,9 +245,16 @@ time_t MemRecursorCache::fakeTTD(MemRecursorCache::OrderedTagIterator_t& entry, if (almostExpired && qname != g_rootdnsname) { if (refresh) { return -1; - } else { + } + else { if (!entry->d_submitted) { - pushAlmostExpiredTask(qname, qtype, entry->d_ttd); + if (qtype == QType::ADDR) { + pushAlmostExpiredTask(qname, QType::A, entry->d_ttd); + pushAlmostExpiredTask(qname, QType::AAAA, entry->d_ttd); + } + else { + pushAlmostExpiredTask(qname, qtype, entry->d_ttd); + } entry->d_submitted = true; } } @@ -257,12 +263,12 @@ time_t MemRecursorCache::fakeTTD(MemRecursorCache::OrderedTagIterator_t& entry, return ttl; } // returns -1 for no hits -time_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType qt, bool requireAuth, vector* res, const ComboAddress& who, bool refresh, const OptTag& routingTag, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth, DNSName* fromAuthZone) +time_t MemRecursorCache::get(time_t now, const DNSName& qname, const QType qt, bool requireAuth, vector* res, const ComboAddress& who, bool refresh, const OptTag& routingTag, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth, DNSName* fromAuthZone) { boost::optional cachedState{boost::none}; uint32_t origTTL; - if(res) { + if (res) { res->clear(); } const uint16_t qtype = qt.getCode(); @@ -290,7 +296,8 @@ time_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType qt, b time_t ttdAAAA = handleHit(*map, entryAAAA, qname, origTTL, res, signatures, authorityRecs, variable, cachedState, wasAuth, fromAuthZone); if (ret > 0) { ret = std::min(ret, ttdAAAA); - } else { + } + else { ret = ttdAAAA; } } @@ -321,7 +328,7 @@ time_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType qt, b if (entries.first != entries.second) { OrderedTagIterator_t firstIndexIterator; - for (auto i=entries.first; i != entries.second; ++i) { + for (auto i = entries.first; i != entries.second; ++i) { firstIndexIterator = map->d_map.project(i); if (i->d_ttd <= now) { @@ -344,7 +351,8 @@ time_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType qt, b *state = *cachedState; } return fakeTTD(firstIndexIterator, qname, qtype, ttd, now, origTTL, refresh); - } else { + } + else { return -1; } } @@ -357,7 +365,7 @@ time_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType qt, b bool found = false; time_t ttd; - for (auto i=entries.first; i != entries.second; ++i) { + for (auto i = entries.first; i != entries.second; ++i) { firstIndexIterator = map->d_map.project(i); if (i->d_ttd <= now) { @@ -386,7 +394,7 @@ time_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType qt, b return -1; } -void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, const DNSName& authZone, boost::optional ednsmask, const OptTag& routingTag, vState state, boost::optional from) +void MemRecursorCache::replace(time_t now, const DNSName& qname, const QType qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, const DNSName& authZone, boost::optional ednsmask, const OptTag& routingTag, vState state, boost::optional from) { auto& mc = getMap(qname); auto map = mc.lock(); @@ -424,16 +432,16 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType qt, } } - time_t maxTTD=std::numeric_limits::max(); - CacheEntry ce=*stored; // this is a COPY - ce.d_qtype=qt.getCode(); + time_t maxTTD = std::numeric_limits::max(); + CacheEntry ce = *stored; // this is a COPY + ce.d_qtype = qt.getCode(); - if(!auth && ce.d_auth) { // unauth data came in, we have some auth data, but is it fresh? - if(ce.d_ttd > now) { // we still have valid data, ignore unauth data + if (!auth && ce.d_auth) { // unauth data came in, we have some auth data, but is it fresh? + if (ce.d_ttd > now) { // we still have valid data, ignore unauth data return; } else { - ce.d_auth = false; // new data won't be auth + ce.d_auth = false; // new data won't be auth } } @@ -467,14 +475,15 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType qt, ce.d_authZone = authZone; if (from) { ce.d_from = *from; - } else { + } + else { ce.d_from = ComboAddress(); } for (const auto& i : content) { /* Yes, we have altered the d_ttl value by adding time(nullptr) to it prior to calling this function, so the TTL actually holds a TTD. */ - ce.d_ttd = min(maxTTD, static_cast(i.d_ttl)); // XXX this does weird things if TTLs differ in the set + ce.d_ttd = min(maxTTD, static_cast(i.d_ttl)); // XXX this does weird things if TTLs differ in the set ce.d_orig_ttl = ce.d_ttd - now; ce.d_records.push_back(i.d_content); } @@ -502,7 +511,8 @@ size_t MemRecursorCache::doWipeCache(const DNSName& name, bool sub, const QType i = idx.erase(i); count++; --mc.d_entriesCount; - } else { + } + else { ++i; } } @@ -523,24 +533,26 @@ size_t MemRecursorCache::doWipeCache(const DNSName& name, bool sub, const QType auto map = mc.lock(); map->d_cachecachevalid = false; auto& idx = map->d_map.get(); - for (auto i = idx.lower_bound(name); i != idx.end(); ) { + for (auto i = idx.lower_bound(name); i != idx.end();) { if (!i->d_qname.isPartOf(name)) break; if (i->d_qtype == qtype || qtype == 0xffff) { count++; i = idx.erase(i); --mc.d_entriesCount; - } else { + } + else { ++i; } } auto& ecsIdx = map->d_ecsIndex.get(); - for (auto i = ecsIdx.lower_bound(name); i != ecsIdx.end(); ) { + for (auto i = ecsIdx.lower_bound(name); i != ecsIdx.end();) { if (!i->d_qname.isPartOf(name)) break; if (i->d_qtype == qtype || qtype == 0xffff) { i = ecsIdx.erase(i); - } else { + } + else { ++i; } } @@ -561,7 +573,7 @@ bool MemRecursorCache::doAgeCache(time_t now, const DNSName& name, const QType q CacheEntry ce = *iter; if (ce.d_ttd < now) - return false; // would be dead anyhow + return false; // would be dead anyhow uint32_t maxTTL = static_cast(ce.d_ttd - now); if (maxTTL > newTTL) { @@ -578,7 +590,7 @@ bool MemRecursorCache::doAgeCache(time_t now, const DNSName& name, const QType q return false; } -bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName &qname, const QType qt, const ComboAddress& who, const OptTag& routingTag, bool requireAuth, vState newState, boost::optional capTTD) +bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName& qname, const QType qt, const ComboAddress& who, const OptTag& routingTag, bool requireAuth, vState newState, boost::optional capTTD) { uint16_t qtype = qt.getCode(); if (qtype == QType::ANY) { @@ -607,7 +619,7 @@ bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName &qname, auto entries = getEntries(*map, qname, qt, routingTag); - for(auto i = entries.first; i != entries.second; ++i) { + for (auto i = entries.first; i != entries.second; ++i) { auto firstIndexIterator = map->d_map.project(i); if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) { @@ -632,8 +644,8 @@ uint64_t MemRecursorCache::doDump(int fd) if (newfd == -1) { return 0; } - auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); - if(!fp) { // dup probably failed + auto fp = std::unique_ptr(fdopen(newfd, "w"), fclose); + if (!fp) { // dup probably failed close(newfd); return 0; } @@ -652,16 +664,16 @@ uint64_t MemRecursorCache::doDump(int fd) try { fprintf(fp.get(), "%s %" PRIu32 " %" PRId64 " IN %s %s ; (%s) auth=%i zone=%s from=%s %s %s\n", i.d_qname.toString().c_str(), i.d_orig_ttl, static_cast(i.d_ttd - now), i.d_qtype.toString().c_str(), j->getZoneRepresentation().c_str(), vStateToString(i.d_state).c_str(), i.d_auth, i.d_authZone.toLogString().c_str(), i.d_from.toString().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str(), !i.d_rtag ? "" : i.d_rtag.get().c_str()); } - catch(...) { + catch (...) { fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); } } - for (const auto &sig : i.d_signatures) { + for (const auto& sig : i.d_signatures) { count++; try { fprintf(fp.get(), "%s %" PRIu32 " %" PRId64 " IN RRSIG %s ; %s\n", i.d_qname.toString().c_str(), i.d_orig_ttl, static_cast(i.d_ttd - now), sig->getZoneRepresentation().c_str(), i.d_netmask.empty() ? "" : i.d_netmask.toString().c_str()); } - catch(...) { + catch (...) { fprintf(fp.get(), "; error printing '%s'\n", i.d_qname.empty() ? "EMPTY" : i.d_qname.toString().c_str()); } } @@ -677,9 +689,10 @@ void MemRecursorCache::doPrune(size_t keep) pruneMutexCollectionsVector(*this, d_maps, keep, cacheSize); } -namespace boost { - size_t hash_value(const MemRecursorCache::OptTag& o) - { - return o ? hash_value(o.get()) : 0xcafebaaf; - } +namespace boost +{ +size_t hash_value(const MemRecursorCache::OptTag& o) +{ + return o ? hash_value(o.get()) : 0xcafebaaf; +} } diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh index c9d0d66ba872..bf5b281425b0 100644 --- a/pdns/recursor_cache.hh +++ b/pdns/recursor_cache.hh @@ -38,6 +38,7 @@ #include #include "iputils.hh" #include "lock.hh" +#include "stat_t.hh" #include "validate.hh" #undef max @@ -51,29 +52,28 @@ public: size_t size() const; size_t bytes(); - pair stats(); + pair stats(); size_t ecsIndexSize(); typedef boost::optional OptTag; - time_t get(time_t, const DNSName &qname, const QType qt, bool requireAuth, vector* res, const ComboAddress& who, bool refresh = false, const OptTag& routingTag = boost::none, vector>* signatures=nullptr, std::vector>* authorityRecs=nullptr, bool* variable=nullptr, vState* state=nullptr, bool* wasAuth=nullptr, DNSName* fromAuthZone=nullptr); + time_t get(time_t, const DNSName& qname, const QType qt, bool requireAuth, vector* res, const ComboAddress& who, bool refresh = false, const OptTag& routingTag = boost::none, vector>* signatures = nullptr, std::vector>* authorityRecs = nullptr, bool* variable = nullptr, vState* state = nullptr, bool* wasAuth = nullptr, DNSName* fromAuthZone = nullptr); - void replace(time_t, const DNSName &qname, const QType qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, const DNSName& authZone, boost::optional ednsmask=boost::none, const OptTag& routingTag = boost::none, vState state=vState::Indeterminate, boost::optional from=boost::none); + void replace(time_t, const DNSName& qname, const QType qt, const vector& content, const vector>& signatures, const std::vector>& authorityRecs, bool auth, const DNSName& authZone, boost::optional ednsmask = boost::none, const OptTag& routingTag = boost::none, vState state = vState::Indeterminate, boost::optional from = boost::none); void doPrune(size_t keep); uint64_t doDump(int fd); - size_t doWipeCache(const DNSName& name, bool sub, QType qtype=0xffff); + size_t doWipeCache(const DNSName& name, bool sub, QType qtype = 0xffff); bool doAgeCache(time_t now, const DNSName& name, QType qtype, uint32_t newTTL); - bool updateValidationStatus(time_t now, const DNSName &qname, QType qt, const ComboAddress& who, const OptTag& routingTag, bool requireAuth, vState newState, boost::optional capTTD); + bool updateValidationStatus(time_t now, const DNSName& qname, QType qt, const ComboAddress& who, const OptTag& routingTag, bool requireAuth, vState newState, boost::optional capTTD); - std::atomic cacheHits{0}, cacheMisses{0}; + pdns::stat_t cacheHits{0}, cacheMisses{0}; private: - struct CacheEntry { - CacheEntry(const boost::tuple& key, bool auth): + CacheEntry(const boost::tuple& key, bool auth) : d_qname(key.get<0>()), d_netmask(key.get<3>().getNormalized()), d_rtag(key.get<2>()), d_state(vState::Indeterminate), d_ttd(0), d_qtype(key.get<1>()), d_auth(auth), d_submitted(false) { } @@ -97,7 +97,7 @@ private: uint32_t d_orig_ttl; QType d_qtype; bool d_auth; - mutable bool d_submitted; // whether this entry has been queued for refetch + mutable bool d_submitted; // whether this entry has been queued for refetch }; /* The ECS Index (d_ecsIndex) keeps track of whether there is any ECS-specific @@ -113,7 +113,8 @@ private: class ECSIndexEntry { public: - ECSIndexEntry(const DNSName& qname, QType qtype): d_nmt(), d_qname(qname), d_qtype(qtype) + ECSIndexEntry(const DNSName& qname, QType qtype) : + d_nmt(), d_qname(qname), d_qtype(qtype) { } @@ -147,66 +148,64 @@ private: QType d_qtype; }; - struct HashedTag {}; - struct SequencedTag {}; - struct NameAndRTagOnlyHashedTag {}; - struct OrderedTag {}; + struct HashedTag + { + }; + struct SequencedTag + { + }; + struct NameAndRTagOnlyHashedTag + { + }; + struct OrderedTag + { + }; typedef multi_index_container< CacheEntry, - indexed_by < - ordered_unique, + indexed_by< + ordered_unique, + composite_key< + CacheEntry, + member, + member, + member, + member>, + composite_key_compare, std::less, std::less>>, + sequenced>, + hashed_non_unique, composite_key< - CacheEntry, - member, - member, - member, - member - >, - composite_key_compare, std::less, std::less > - >, - sequenced >, - hashed_non_unique, - composite_key< - CacheEntry, - member, - member - > - > - > - > cache_t; + CacheEntry, + member, + member>>>> + cache_t; typedef MemRecursorCache::cache_t::index::type::iterator OrderedTagIterator_t; typedef MemRecursorCache::cache_t::index::type::iterator NameAndRTagOnlyHashedTagIterator_t; typedef multi_index_container< ECSIndexEntry, - indexed_by < - hashed_unique , - composite_key< - ECSIndexEntry, - member, - member - > - >, + indexed_by< + hashed_unique, + composite_key< + ECSIndexEntry, + member, + member>>, ordered_unique, - composite_key< - ECSIndexEntry, - member, - member - >, - composite_key_compare > - > - > - > ecsIndex_t; + composite_key< + ECSIndexEntry, + member, + member>, + composite_key_compare>>>> + ecsIndex_t; typedef std::pair Entries; struct MapCombo { MapCombo() {} - MapCombo(const MapCombo &) = delete; - MapCombo & operator=(const MapCombo &) = delete; + MapCombo(const MapCombo&) = delete; + MapCombo& operator=(const MapCombo&) = delete; struct LockedContent { cache_t d_map; @@ -224,7 +223,7 @@ private: } }; - std::atomic d_entriesCount{0}; + pdns::stat_t d_entriesCount{0}; LockGuardedTryHolder lock() { @@ -242,7 +241,7 @@ private: }; vector d_maps; - MapCombo& getMap(const DNSName &qname) + MapCombo& getMap(const DNSName& qname) { return d_maps.at(qname.hash() % d_maps.size()); } @@ -250,8 +249,8 @@ private: static time_t fakeTTD(OrderedTagIterator_t& entry, const DNSName& qname, QType qtype, time_t ret, time_t now, uint32_t origTTL, bool refresh); bool entryMatches(OrderedTagIterator_t& entry, QType qt, bool requireAuth, const ComboAddress& who); - Entries getEntries(MapCombo::LockedContent& content, const DNSName &qname, const QType qt, const OptTag& rtag); - cache_t::const_iterator getEntryUsingECSIndex(MapCombo::LockedContent& content, time_t now, const DNSName &qname, QType qtype, bool requireAuth, const ComboAddress& who); + Entries getEntries(MapCombo::LockedContent& content, const DNSName& qname, const QType qt, const OptTag& rtag); + cache_t::const_iterator getEntryUsingECSIndex(MapCombo::LockedContent& content, time_t now, const DNSName& qname, QType qtype, bool requireAuth, const ComboAddress& who); time_t handleHit(MapCombo::LockedContent& content, OrderedTagIterator_t& entry, const DNSName& qname, uint32_t& origTTL, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, boost::optional& state, bool* wasAuth, DNSName* authZone); @@ -273,6 +272,7 @@ public: } }; -namespace boost { - size_t hash_value(const MemRecursorCache::OptTag& rtag); +namespace boost +{ +size_t hash_value(const MemRecursorCache::OptTag& rtag); } diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am index e9b9b43a4ae8..d1829d4b179f 100644 --- a/pdns/recursordist/Makefile.am +++ b/pdns/recursordist/Makefile.am @@ -105,6 +105,7 @@ pdns_recursor_SOURCES = \ capabilities.cc capabilities.hh \ circular_buffer.hh \ comment.hh \ + credentials.cc credentials.hh \ dns.hh dns.cc \ dns_random.hh dns_random.cc \ dnsbackend.hh \ @@ -158,10 +159,13 @@ pdns_recursor_SOURCES = \ query-local-address.hh query-local-address.cc \ rcpgenerator.cc rcpgenerator.hh \ rec-carbon.cc \ + rec-eventtrace.cc rec-eventtrace.hh \ rec-lua-conf.hh rec-lua-conf.cc \ rec-protozero.cc rec-protozero.hh \ rec-snmp.hh rec-snmp.cc \ rec-taskqueue.cc rec-taskqueue.hh \ + rec-tcpout.cc rec-tcpout.hh \ + rec-zonetocache.cc rec-zonetocache.hh \ rec_channel.cc rec_channel.hh rec_metrics.hh \ rec_channel_rec.cc \ recpacketcache.cc recpacketcache.hh \ @@ -221,6 +225,8 @@ pdns_recursor_LDADD = \ $(PROBDS_LIBS) \ $(LIBCAP_LIBS) +rec_control_LDADD = $(LIBCRYPTO_LIBS) + pdns_recursor_LDFLAGS = $(AM_LDFLAGS) \ $(LIBCRYPTO_LDFLAGS) $(BOOST_CONTEXT_LDFLAGS) \ $(BOOST_SYSTEM_LDFLAGS) @@ -240,6 +246,7 @@ testrunner_SOURCES = \ base32.cc \ base64.cc base64.hh \ circular_buffer.hh \ + credentials.cc credentials.hh \ dns.cc dns.hh \ dns_random.cc dns_random.hh \ dnslabeltext.cc \ @@ -271,6 +278,8 @@ testrunner_SOURCES = \ qtype.cc qtype.hh \ query-local-address.hh query-local-address.cc \ rcpgenerator.cc \ + rec-eventtrace.cc rec-eventtrace.hh \ + rec-zonetocache.cc rec-zonetocache.hh \ recpacketcache.cc recpacketcache.hh \ recursor_cache.cc recursor_cache.hh \ resolver.hh resolver.cc \ @@ -290,6 +299,7 @@ testrunner_SOURCES = \ test-base32_cc.cc \ test-base64_cc.cc \ test-common.hh \ + test-credentials_cc.cc \ test-dns_random_hh.cc \ test-dnsname_cc.cc \ test-dnsparser_hh.cc \ @@ -307,6 +317,7 @@ testrunner_SOURCES = \ test-negcache_cc.cc \ test-packetcache_hh.cc \ test-rcpgenerator_cc.cc \ + test-rec-zonetocache.cc \ test-recpacketcache_cc.cc \ test-recursorcache_cc.cc \ test-rpzloader_cc.cc \ @@ -365,11 +376,16 @@ if LIBSODIUM pdns_recursor_SOURCES += \ sodiumsigners.cc pdns_recursor_LDADD += $(LIBSODIUM_LIBS) + +rec_control_LDADD += $(LIBSODIUM_LIBS) + testrunner_SOURCES += \ sodiumsigners.cc testrunner_LDADD += $(LIBSODIUM_LIBS) endif + + if LIBDECAF pdns_recursor_SOURCES += \ decafsigners.cc @@ -431,14 +447,23 @@ AM_CPPFLAGS += $(LIBSSL_CFLAGS) pdns_recursor_LDADD += $(LIBSSL_LIBS) endif -if HAVE_GNUTLS -AM_CPPFLAGS += $(GNUTLS_CFLAGS) -pdns_recursor_LDADD += $(GNUTLS_LIBS) +#if HAVE_GNUTLS +#AM_CPPFLAGS += $(GNUTLS_CFLAGS) +#pdns_recursor_LDADD += $(GNUTLS_LIBS) +#endif endif +if HAVE_LIBCURL +pdns_recursor_SOURCES += minicurl.cc minicurl.hh +pdns_recursor_LDADD += $(LIBCURL) +testrunner_SOURCES += minicurl.cc minicurl.hh +testrunner_LDADD += $(LIBCURL) endif + rec_control_SOURCES = \ arguments.cc arguments.hh \ + base64.cc base64.hh \ + credentials.cc credentials.hh \ dnslabeltext.cc \ dnsname.hh dnsname.cc \ logger.cc \ @@ -520,8 +545,10 @@ if HAVE_SYSTEMD pdns-recursor.service: pdns-recursor.service.in $(AM_V_GEN)sed -e 's![@]sbindir[@]!$(sbindir)!' -e 's![@]service_user[@]!$(service_user)!' -e 's![@]service_group[@]!$(service_group)!' < $< > $@ if !HAVE_SYSTEMD_WITH_RUNTIME_DIR_ENV +if HAVE_SYSTEMD_PERCENT_T $(AM_V_GEN)sed -e 's!/pdns_recursor!& --socket-dir=%t/pdns-recursor!' -i $@ endif +endif if !HAVE_SYSTEMD_LOCK_PERSONALITY $(AM_V_GEN)perl -ni -e 'print unless /^LockPersonality/' $@ endif @@ -580,11 +607,14 @@ endif pdns-recursor@.service: pdns-recursor.service $(AM_V_GEN)sed -e 's!/pdns_recursor!& --config-name=%i!' \ -e 's!Recursor!& %i!' \ - -e 's!RuntimeDirectory=.*!&-%i!' \ + -e 's!SyslogIdentifier=.*!&-%i!' \ < $< > $@ +if HAVE_SYSTEMD_PERCENT_T + $(AM_V_GEN)sed -e 's!RuntimeDirectory=.*!&-%i!' -i $@ if !HAVE_SYSTEMD_WITH_RUNTIME_DIR_ENV $(AM_V_GEN)sed -e 's!--socket-dir=[^ ]\+!&-%i !' -i $@ endif +endif systemdsystemunitdir = $(SYSTEMD_DIR) diff --git a/pdns/recursordist/RECURSOR-MIB.txt b/pdns/recursordist/RECURSOR-MIB.txt index b2478d7bfed8..22773d254954 100644 --- a/pdns/recursordist/RECURSOR-MIB.txt +++ b/pdns/recursordist/RECURSOR-MIB.txt @@ -39,6 +39,12 @@ rec MODULE-IDENTITY REVISION "202107200000Z" DESCRIPTION "Added almost expired task metrics." + REVISION "202110270000Z" + DESCRIPTION "Added more UDP errors metric." + + REVISION "202111090000Z" + DESCRIPTION "Added NOTIFY-related metrics." + ::= { powerdns 2 } powerdns OBJECT IDENTIFIER ::= { enterprises 43315 } @@ -981,6 +987,69 @@ almostExpiredExceptions OBJECT-TYPE "Number of almost-expired tasks that caused an exception" ::= { stats 117 } +udpInCsumErrors OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of UDP in checksum errors (Linux only)" + ::= { stats 118 } + +udp6RecvbufErrors OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of UDP6 recvbuf errors (Linux only)" + ::= { stats 119 } + +udp6SndbufErrors OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of UDP6 sndbuf errors (Linux only)" + ::= { stats 120 } + +udp6NoportErrors OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of UDP6 noport errors (Linux only)" + ::= { stats 121 } + +udp6inErrors OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of UDP6 in errors (Linux only)" + ::= { stats 122 } + +udp6InCsumErrors OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of UDP6 in checksum errors (Linux only)" + ::= { stats 123 } + +sourceDisallowedNotify OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of NOTIFY operations not allowed by allow-notify-from" + ::= { stats 124 } + +zoneDisallowedNotify OBJECT-TYPE + SYNTAX Counter64 + MAX-ACCESS read-only + STATUS current + DESCRIPTION + "Number of NOTIFY operations not allowed by allow-notify-for" + ::= { stats 125 } --- --- Traps / Notifications --- @@ -1142,7 +1211,16 @@ recGroup OBJECT-GROUP aggressiveNSECCacheNSECWcHits, aggressiveNSECCacheNSEC3WcHits, dotOutqueries, - dns64PrefixAnswers + dns64PrefixAnswers, + almostExpiredPushed, + almostExpiredRun, + almostExpiredExceptions, + udpInCsumErrors, + udp6RecvbufErrors, + udp6SndbufErrors, + udp6NoportErrors, + udp6inErrors, + udp6InCsumErrors } STATUS current DESCRIPTION "Objects conformance group for PowerDNS Recursor" diff --git a/pdns/recursordist/aggressive_nsec.cc b/pdns/recursordist/aggressive_nsec.cc index 9368e4a865fa..2f332b639542 100644 --- a/pdns/recursordist/aggressive_nsec.cc +++ b/pdns/recursordist/aggressive_nsec.cc @@ -725,8 +725,14 @@ bool AggressiveNSECCache::getNSEC3Denial(time_t now, std::shared_ptr>>& zones); SharedLockGuarded>>> d_zones; - std::atomic d_nsecHits{0}; - std::atomic d_nsec3Hits{0}; - std::atomic d_nsecWildcardHits{0}; - std::atomic d_nsec3WildcardHits{0}; - std::atomic d_entriesCount{0}; + pdns::stat_t d_nsecHits{0}; + pdns::stat_t d_nsec3Hits{0}; + pdns::stat_t d_nsecWildcardHits{0}; + pdns::stat_t d_nsec3WildcardHits{0}; + pdns::stat_t d_entriesCount{0}; uint64_t d_maxEntries{0}; }; diff --git a/pdns/recursordist/configure.ac b/pdns/recursordist/configure.ac index 2727cbb51773..c86b941a193c 100644 --- a/pdns/recursordist/configure.ac +++ b/pdns/recursordist/configure.ac @@ -85,20 +85,23 @@ PDNS_CHECK_LIBCRYPTO_EDDSA PDNS_WITH_LIBSODIUM PDNS_WITH_LIBDECAF PDNS_WITH_LIBCAP +PDNS_CHECK_LIBCURL PDNS_WITH_NET_SNMP -AM_CONDITIONAL([HAVE_GNUTLS], [false]) +# not runtime selectable at the moment +#AM_CONDITIONAL([HAVE_GNUTLS], [false]) AM_CONDITIONAL([HAVE_LIBSSL], [false]) PDNS_ENABLE_DNS_OVER_TLS AS_IF([test "x$enable_dns_over_tls" != "xno"], [ PDNS_WITH_LIBSSL - PDNS_WITH_GNUTLS + # not runtime selectable at the moment + # PDNS_WITH_GNUTLS - AS_IF([test "x$HAVE_GNUTLS" != "x1" -a "x$HAVE_LIBSSL" != "x1"], [ - AC_MSG_ERROR([DNS over TLS support requested but neither GnuTLS nor OpenSSL are available]) + AS_IF([test "x$HAVE_LIBSSL" != "x1"], [ + AC_MSG_ERROR([DNS over TLS support requested but no OpenSSL available]) ]) ]) @@ -242,17 +245,21 @@ AS_IF([test "x$enable_dns_over_tls" != "xno"], [AC_MSG_NOTICE([DNS over TLS: yes])], [AC_MSG_NOTICE([DNS over TLS: no])] ) -AS_IF([test "x$enable_dns_over_tls" != "xno"], [ - AS_IF([test "x$GNUTLS_LIBS" != "x"], - [AC_MSG_NOTICE([GnuTLS: yes])], - [AC_MSG_NOTICE([GnuTLS: no])] - )] -) +#AS_IF([test "x$enable_dns_over_tls" != "xno"], [ +# AS_IF([test "x$GNUTLS_LIBS" != "x"], +# [AC_MSG_NOTICE([GnuTLS: yes])], +# [AC_MSG_NOTICE([GnuTLS: no])] +# )] +#) AS_IF([test "x$enable_dns_over_tls" != "xno"], [ AS_IF([test "x$LIBSSL_LIBS" != "x"], [AC_MSG_NOTICE([OpenSSL: yes])], [AC_MSG_NOTICE([OpenSSL: no])] )] ) +AS_IF([test "x$HAVE_LIBCURL" != "xn"], + [AC_MSG_NOTICE([libcurl: yes])], + [AC_MSG_NOTICE([libcurl: no])] +) AC_MSG_NOTICE([Context library: $pdns_context_library]) AC_MSG_NOTICE([]) diff --git a/pdns/recursordist/contrib/kv-example-script.lua b/pdns/recursordist/contrib/kv-example-script.lua index 18c402f5b456..63be3972921e 100644 --- a/pdns/recursordist/contrib/kv-example-script.lua +++ b/pdns/recursordist/contrib/kv-example-script.lua @@ -1,58 +1,58 @@ ---[[ +--[[ This implements a two-step domain filtering solution where the status of an IP address and a domain name need to be looked up. To do so, we use the udpQuestionResponse answers which generically allows us to do asynchronous lookups via UDP. Such lookups can be slow, but they won't block PowerDNS while we wait for them. -To benefit from this hook, +To benefit from this hook, .. To test, use the 'kvresp' example program provided. --]] function preresolve (dq) - print ("preresolve handler called for: "..dq.remoteaddr:toString().. ", local: ".. dq.localaddr:toString()..", ".. dq.qname:toString()..", ".. dq.qtype) - dq.followupFunction="udpQueryResponse" - dq.udpCallback="gotdomaindetails" - dq.udpQueryDest=newCA("127.0.0.1:5555") - dq.udpQuery = "DOMAIN "..dq.qname:toString() - return true; + pdnslog("preresolve handler called for: "..dq.remoteaddr:toString()..", local: "..dq.localaddr:toString()..", "..dq.qname:toString()..", "..dq.qtype) + dq.followupFunction="udpQueryResponse" + dq.udpCallback="gotdomaindetails" + dq.udpQueryDest=newCA("127.0.0.1:5555") + dq.udpQuery = "DOMAIN "..dq.qname:toString() + return true; end function gotdomaindetails(dq) - print("gotdomaindetails called, got: "..dq.udpAnswer) - if(dq.udpAnswer == "0") - then - print("This domain needs no filtering, not looking up this domain") - dq.followupFunction="" - return false - end - print("Domain might need filtering for some users") - dq.variable = true -- disable packet cache - local data={} - data["domaindetails"]= dq.udpAnswer - dq.data=data - dq.udpQuery="IP "..dq.remoteaddr:toString() - dq.udpCallback="gotipdetails" - print("returning true in gotipdetails") - return true -end + pdnslog("gotdomaindetails called, got: "..dq.udpAnswer) -function gotipdetails(dq) + if(dq.udpAnswer == "0") + then + pdnslog("This domain needs no filtering, not looking up this domain") dq.followupFunction="" - print("So status of IP is "..dq.udpAnswer.." and status of domain is "..dq.data.domaindetails) - if(dq.data.domaindetails=="1" and dq.udpAnswer=="1") - then - print("IP wants filtering and domain is of the filtered kind") - dq:addAnswer(pdns.CNAME, "blocked.powerdns.com") - return true - else - print("Returning false (normal resolution should proceed, for this user)") - return false - end + return false + end + pdnslog("Domain might need filtering for some users") + dq.variable = true -- disable packet cache + + local data={} + data["domaindetails"]= dq.udpAnswer + dq.data=data + dq.udpQuery="IP "..dq.remoteaddr:toString() + dq.udpCallback="gotipdetails" + pdnslog("returning true in gotipdetails") + return true end - - +function gotipdetails(dq) + dq.followupFunction="" + pdnslog("So status of IP is "..dq.udpAnswer.." and status of domain is "..dq.data.domaindetails) + + if(dq.data.domaindetails=="1" and dq.udpAnswer=="1") + then + pdnslog("IP wants filtering and domain is of the filtered kind") + dq:addAnswer(pdns.CNAME, "blocked.powerdns.com") + return true + else + pdnslog("Returning false (normal resolution should proceed, for this user)") + return false + end +end diff --git a/pdns/recursordist/contrib/powerdns-example-script.lua b/pdns/recursordist/contrib/powerdns-example-script.lua index a22c9ed0a06f..adebcb033073 100644 --- a/pdns/recursordist/contrib/powerdns-example-script.lua +++ b/pdns/recursordist/contrib/powerdns-example-script.lua @@ -18,63 +18,63 @@ magicMetric = getMetric("magic") -- return false to say you did not take over the question, but we'll still listen to 'variable' -- to selectively disable the cache function preresolve(dq) - print("Got question for "..dq.qname:toString().." from "..dq.remoteaddr:toString().." to "..dq.localaddr:toString()) + pdnslog("Got question for "..dq.qname:toString().." from "..dq.remoteaddr:toString().." to "..dq.localaddr:toString()) local ednssubnet=dq:getEDNSSubnet() if(ednssubnet) then - print("Packet EDNS subnet source: "..ednssubnet:toString()..", "..ednssubnet:getNetwork():toString()) + pdnslog("Packet EDNS subnet source: "..ednssubnet:toString()..", "..ednssubnet:getNetwork():toString()) end - + local a=dq:getEDNSOption(3) if(a) then - print("There is an EDNS option 3 present: "..a) + pdnslog("There is an EDNS option 3 present: "..a) end loc = newCA("127.0.0.1") if(dq.remoteaddr:equal(loc)) then - print("Query from loopback") + pdnslog("Query from loopback") end -- note that the comparisons below are CaSe InSensiTivE and you don't have to worry about trailing dots if(dq.qname:equal("magic.com")) then magicMetric:inc() - print("Magic!") + pdnslog("Magic!") else - print("not magic..") + pdnslog("not magic..") end if(dq.qname:__eq(magic2)) -- we hope to improve this syntax then - print("Faster magic") -- compares against existing DNSName + pdnslog("Faster magic") -- compares against existing DNSName end -- sadly, dq.qname == magic2 won't work yet - + if blockset:check(dq.qname) then dq.variable = true -- disable packet cache in any case if dq.qtype == pdns.A then dq:addAnswer(pdns.A, "1.2.3.4") - dq:addAnswer(pdns.TXT, "\"Hello!\"", 3601) -- ttl + dq:addAnswer(pdns.TXT, "\"Hello!\"", 3601) -- ttl return true; end end - + if dropset:check(dq.qname) then - dq.rcode = pdns.DROP + dq.rcode = pdns.DROP return true; end - - + + if malwareset:check(dq.qname) then dq:addAnswer(pdns.CNAME, "xs.powerdns.com.") dq.rcode = 0 dq.followupFunction="followCNAMERecords" -- this makes PowerDNS lookup your CNAME return true; - end - - return false; + end + + return false; end @@ -87,13 +87,13 @@ function nodata(dq) dq.followupPrefix="fe80::" return true end - + if dq.qtype == pdns.PTR then dq.followupFunction="getFakePTRRecords" dq.followupName=dq.qname dq.followupPrefix="fe80::" return true - end + end return false end @@ -103,22 +103,22 @@ badips:addMask("127.1.0.0/16") -- this check is applied before any packet parsing is done function ipfilter(rem, loc, dh) - print("ipfilter called, rem: ", rem:toStringWithPort(), "loc: ",loc:toStringWithPort(),"match:", badips:match(rem)) - print("id: ",dh:getID(), "aa: ", dh:getAA(), "ad: ", dh:getAD(), "arcount: ", dh:getARCOUNT()) - print("ports: ",rem:getPort(),loc:getPort()) + pdnslog("ipfilter called, rem: "..rem:toStringWithPort().."loc: "..loc:toStringWithPort().."match:"..badips:match(rem)) + pdnslog("id: "..dh:getID().."aa: "..dh:getAA().."ad: "..dh:getAD().."arcount: "..dh:getARCOUNT()) + pdnslog("ports: "..rem:getPort()..loc:getPort()) return badips:match(rem) end -- postresolve runs after the packet has been answered, and can be used to change things -- or still drop function postresolve(dq) - print("postresolve called for ",dq.qname:toString()) + pdnslog("postresolve called for "..dq.qname:toString()) local records = dq:getRecords() for k,v in pairs(records) do - print(k, v.name:toString(), v:getContent()) + pdnslog(k, v.name:toString()..v:getContent()) if v.type == pdns.A and v:getContent() == "185.31.17.73" then - print("Changing content!") + pdnslog("Changing content!") v:changeContent("130.161.252.29") v.ttl=1 end @@ -130,7 +130,7 @@ end nxdomainsuffix=newDN("com") function nxdomain(dq) - print("Hooking: ",dq.qname:toString()) + pdnslog("Hooking: "..dq.qname:toString()) if dq.qname:isPartOf(nxdomainsuffix) then dq.rcode=0 -- make it a normal answer @@ -139,4 +139,4 @@ function nxdomain(dq) return true end return false -end \ No newline at end of file +end diff --git a/pdns/recursordist/credentials.cc b/pdns/recursordist/credentials.cc new file mode 120000 index 000000000000..385b2128dfdc --- /dev/null +++ b/pdns/recursordist/credentials.cc @@ -0,0 +1 @@ +../credentials.cc \ No newline at end of file diff --git a/pdns/recursordist/credentials.hh b/pdns/recursordist/credentials.hh new file mode 120000 index 000000000000..2c60bfff8f49 --- /dev/null +++ b/pdns/recursordist/credentials.hh @@ -0,0 +1 @@ +../credentials.hh \ No newline at end of file diff --git a/pdns/recursordist/docs/changelog/4.4.rst b/pdns/recursordist/docs/changelog/4.4.rst index 034ef86a3681..2b729a3e75a4 100644 --- a/pdns/recursordist/docs/changelog/4.4.rst +++ b/pdns/recursordist/docs/changelog/4.4.rst @@ -1,6 +1,49 @@ Changelogs for 4.4.x ==================== +.. changelog:: + :version: 4.4.7 + :released: 5th of November 2021 + + .. change:: + :tags: Bug Fixes + :pullreq: 10910 + :tickets: 10908 + + A SHA-384 DS should not trump a SHA-256 one, only potentially ignore SHA-1 DS records. + + .. change:: + :tags: Bug Fixes + :pullreq: 10909 + :tickets: 10905 + + rec_control wipe-cache-typed should check if a qtype argument is present and valid. + +.. changelog:: + :version: 4.4.6 + :released: 8th of October 2021 + + .. change:: + :tags: Bug Fixes + :pullreq: 10802 + :tickets: 10768 + + Use the correct RPZ policy name for statistics when loading via XFR. + + .. change:: + :tags: Bug Fixes + :pullreq: 10654 + :tickets: 10643 + + NS from the cache could be a forwarder, take that into account for throttling decision. + + .. change:: + :tags: Bug Fixes + :pullreq: 10628 + :tickets: 10627 + + Check in more places if the policy has been updated before using or modifying it. + .. changelog:: :version: 4.4.5 :released: 30th of July 2021 diff --git a/pdns/recursordist/docs/changelog/4.5.rst b/pdns/recursordist/docs/changelog/4.5.rst index f674fa78d19d..371f9c5bdbf5 100644 --- a/pdns/recursordist/docs/changelog/4.5.rst +++ b/pdns/recursordist/docs/changelog/4.5.rst @@ -1,5 +1,96 @@ Changelogs for 4.5.X ==================== +.. changelog:: + :version: 4.5.7 + :released: 5th of November 2021 + + .. change:: + :tags: Bug Fixes + :pullreq: 10912 + :tickets: 10908 + + A SHA-384 DS should not trump a SHA-256 one, only potentially ignore SHA-1 DS records. + + .. change:: + :tags: Bug Fixes + :pullreq: 10911 + :tickets: 10905 + + rec_control wipe-cache-typed should check if a qtype arg is present and valid. + + .. change:: + :tags: Bug Fixes + :pullreq: 10863 + :tickets: 10842 + + Put the correct string into appliedPolicyTrigger for Netmask matching rules. + +.. changelog:: + :version: 4.5.6 + :released: 11th of October 2021 + + .. change:: + :tags: Bug Fixes + :pullreq: 10806 + :tickets: 10565 + + Do not use DNSKEYs found below an apex for validation. + + .. change:: + :tags: Bug Fixes + :pullreq: 10807 + :tickets: 10622 + + Detect a loop when the denial of the DS comes from the child zone. + + .. change:: + :tags: Bug Fixes + :pullreq: 10809 + :tickets: 10632 + + Match ordering of PacketID using the Birthday vs non-Birthday comparator. + + .. change:: + :tags: Bug Fixes + :pullreq: 10811 + :tickets: 10633 + + Pass the Lua context to follow up queries (follow CNAME, dns64). + + .. change:: + :tags: Bug Fixes + :pullreq: 10813 + :tickets: 10718 + + Only the DNAME records are authoritative in DNAME answers. + + .. change:: + :tags: Bug Fixes + :pullreq: 10803 + :tickets: 10768 + + Use the correct RPZ policy name for statistics when loading via XFR. + + .. change:: + :tags: Bug Fixes + :pullreq: 10717 + :tickets: 10701 + + Fix the aggressive cache returning duplicated NSEC3 records. + + .. change:: + :tags: Bug Fixes + :pullreq: 10655 + :tickets: 10643 + + NS from the cache could be a forwarder, take that into account for throttling decision. + + .. change:: + :tags: Bug Fixes + :pullreq: 10629 + :tickets: 10627 + + Check in more places if the policy has been updated before using or modifying it. .. changelog:: :version: 4.5.5 diff --git a/pdns/recursordist/docs/changelog/4.6.rst b/pdns/recursordist/docs/changelog/4.6.rst new file mode 100644 index 000000000000..4ff5b16dd785 --- /dev/null +++ b/pdns/recursordist/docs/changelog/4.6.rst @@ -0,0 +1,262 @@ +Changelogs for 4.6.X +==================== + +.. changelog:: + :version: 4.6.0-beta1 + :released: 9th of November 2021 + + .. change:: + :tags: Improvements + :pullreq: 10865 + + Return documented reply on /api/v1 access. + + .. change:: + :tags: Bug Fixes + :pullreq: 10943 + :tickets: 10938 + + Credentials: EVP_PKEY_CTX_set1_scrypt_salt() takes an `unsigned char*`. + + .. change:: + :tags: Improvements + :pullreq: 10919 + :tickets: 10852 + + Add more UDP error metrics (checksum, IPv6). + + .. change:: + :tags: Improvements + :pullreq: 10930,10965 + + Move to a stream based socket for the control channel. + + .. change:: + :tags: Improvements + :pullreq: 10901 + + ZoneParserTNG: Stricter checks when loading a zone file. + + .. change:: + :tags: Bug Fixes + :pullreq: 10926 + + Fix regression of carbon-ourname. + + .. change:: + :tags: Improvements + :pullreq: 10891 + + Implement fd-usage metric for OpenBSD. + +.. changelog:: + :version: 4.6.0-alpha2 + :released: 25th of October 2021 + + .. change:: + :tags: Improvements + :pullreq: 10646,10868,10870 + + Move to modern C++ constructs (Rosen Penev). + + .. change:: + :tags: Bug Fixes + :pullreq: 10842 + + Correct appliedPolicyTrigger value for IP matches. + + .. change:: + :tags: Improvements + :pullreq: 10843 + + NOD - use structured logging API. + + .. change:: + :tags: Improvements + :pullreq: 10847 + + Sync dnsmessage.proto. + + .. change:: + :tags: Improvements + :pullreq: 10567 + :tickets: 7558,7420 + + Introduce experimental Event Trace function to get a more detailed view the work done by the Recursor. + + .. change:: + :tags: Improvements + :pullreq: 10797 + :tickets: 9135 + + Use packetcache-servfail-ttl for all packet cache entries considered an error reply. + + .. change:: + :tags: Improvements + :pullreq: 10505,10794,10799 + + Add a periodic zones-to-cache function. + + .. change:: + :tags: Bug Fixes + :pullreq: 10768 + + Use the correct RPZ policy name when loading via XFR. + + .. change:: + :tags: Bug Fixes + :pullreq: 10760 + + Don't create file with wide permissions. + + .. change:: + :tags: Bug Fixes + :pullreq: 10757 + + Update the stats (serial, number of records, timestamp) for RPZ files. + + + +.. changelog:: + :version: 4.6.0-alpha1 + :released: 29th of September 2021 + + .. change:: + :tags: Improvements + :pullreq: 10669 + + TCP/DoT outgoing connection pooling. + + .. change:: + :tags: Bug Fixes + :pullreq: 10718 + :tickets: 10713 + + Only the DNAME records are authoritative in DNAME answers. + + .. change:: + :tags: Improvements + :pullreq: 10599 + + Be more strict when validating DS with respect to parent/child NSEC(3)s. + + .. change:: + :tags: Bug Fixes + :pullreq: 10633 + :tickets: 10632 + + Pass the Lua context to follow up queries (follow CNAME, dns64). + + .. change:: + :tags: Improvements + :pullreq: 10605 + :tickets: 10554 + + Keep a count of per RPZ (or filter) hits. + + .. change:: + :tags: Bug Fixes + :pullreq: 10622 + :tickets: 10621 + + Detect a loop when the denial of the DS comes from the child zone. + + .. change:: + :tags: Improvements + :pullreq: 10554,10738 + :tickets: 10735 + + Modify per-thread cpu usage stats to be Prometheus-friendly. + + .. change:: + :tags: Improvements + :pullreq: 10598 + + Refactor almost-expired code and add more detailed stats. + + .. change:: + :tags: Improvements + :pullreq: 10546 + + Add dns64 metrics. + + .. change:: + :tags: Bug Fixes + :pullreq: 10602 + + Process policy and potential Drop action after Lua hooks. + + .. change:: + :tags: Improvements + :pullreq: 10634 + :tickets: 10631 + + Move MacOS to kqueue event handler and assorted compile fixes. + + .. change:: + :tags: Bug Fixes + :pullreq: 10565 + + Do not use DNSKEYs found below an apex for validation. + + .. change:: + :tags: Improvements + :pullreq: 10122,10663 + :tickets: 9077,10122 + + Cumulative and Prometheus friendly histograms. + + .. change:: + :tags: Improvements + :pullreq: 10428,10659,10533 + + Rewrite of outgoing TCP code and implement DoT to auth or forwarders. + + .. change:: + :tags: Improvements + :pullreq: 10467 + + Switch OpenBSD to kqueue event handler. + + .. change:: + :tags: Improvements + :pullreq: 10396 + :tickets: 10395 + + Take into account g_quiet when determining loglevel and change a few loglevels. + + .. change:: + :tags: Improvements + :pullreq: 10349,10623 + + Move to tcpiohandler for outgoing TCP, sharing much more code with dnsdist. + + .. change:: + :tags: Improvements + :pullreq: 10288 + + Deprecate offensive setting names. + + .. change:: + :tags: Improvements + :pullreq: 10160 + + Implement structured logging API. + + .. change:: + :tags: Improvements + :pullreq: 10264 + + Disable PMTU for IPv6. + + .. change:: + :tags: Improvements + :pullreq: 10157 + + Move to hashed passwords for the web interface. + + .. change:: + :tags: Improvements + :pullreq: 10491 + + Rec: Add bindings to set arbitrary key-value metadata in logged messages diff --git a/pdns/recursordist/docs/changelog/index.rst b/pdns/recursordist/docs/changelog/index.rst index ea5fa306587e..77d7f42c076d 100644 --- a/pdns/recursordist/docs/changelog/index.rst +++ b/pdns/recursordist/docs/changelog/index.rst @@ -6,6 +6,7 @@ The changelogs for the recursor are split between release trains. .. toctree:: :maxdepth: 2 + 4.6 4.5 4.4 4.3 diff --git a/pdns/recursordist/docs/http-api/endpoint-servers-config.rst b/pdns/recursordist/docs/http-api/endpoint-servers-config.rst index 6fe6d88333be..713402d57759 100644 --- a/pdns/recursordist/docs/http-api/endpoint-servers-config.rst +++ b/pdns/recursordist/docs/http-api/endpoint-servers-config.rst @@ -5,7 +5,7 @@ Change a single setting .. note:: - Only :ref:`setting-allow-from` can be set. + Only :ref:`setting-allow-from` and :ref:`setting-allow-notify-from` can be set. :param server_id: The name of the server :param config_setting_name: The name of the setting to change diff --git a/pdns/recursordist/docs/http-api/index.rst b/pdns/recursordist/docs/http-api/index.rst index e13b2ec21fe2..b7e3efad46d0 100644 --- a/pdns/recursordist/docs/http-api/index.rst +++ b/pdns/recursordist/docs/http-api/index.rst @@ -28,7 +28,7 @@ The following webserver related configuration items are available: * :ref:`setting-webserver`: If set to anything but 'no', a webserver is launched. * :ref:`setting-webserver-address`: Address to bind the webserver to. Defaults to 127.0.0.1, which implies that only the local computer is able to connect to the nameserver! To allow remote hosts to connect, change to 0.0.0.0 or the physical IP address of your nameserver. -* :ref:`setting-webserver-password`: If set, viewers will have to enter this plaintext password in order to gain access to the statistics. +* :ref:`setting-webserver-password`: If set, viewers will have to enter this password in order to gain access to the statistics. * :ref:`setting-webserver-port`: Port to bind the webserver to. * :ref:`setting-webserver-allow-from`: Netmasks that are allowed to connect to the webserver diff --git a/pdns/recursordist/docs/lua-config/index.rst b/pdns/recursordist/docs/lua-config/index.rst index ab2950078316..44d04d161711 100644 --- a/pdns/recursordist/docs/lua-config/index.rst +++ b/pdns/recursordist/docs/lua-config/index.rst @@ -1,5 +1,5 @@ -Lua Configuration: Trustanchors, Query Logging, RPZs and Sortlist -================================================================= +Lua Configuration: Trustanchors, Query Logging, RPZs, Sortlist and Zone to Cache +================================================================================ Since version 4.0.0, the PowerDNS Recursor supports additional configuration options that have to be loaded through :ref:`setting-lua-config-file`. @@ -9,5 +9,6 @@ Since version 4.0.0, the PowerDNS Recursor supports additional configuration opt protobuf rpz sortlist + ztc In addition, :func:`pdnslog` together with ``pdns.loglevels`` is also supported in the Lua configuration file. diff --git a/pdns/recursordist/docs/lua-config/rpz.rst b/pdns/recursordist/docs/lua-config/rpz.rst index 11450e8884e6..4efe0d73232f 100644 --- a/pdns/recursordist/docs/lua-config/rpz.rst +++ b/pdns/recursordist/docs/lua-config/rpz.rst @@ -70,7 +70,25 @@ As of version 4.2.0, the first parameter of :func:`rpzPrimary` can be a list of In the example above, two addresses are specified and will be tried one after another until a response is obtained. The first address uses the default port (53) while the second one uses port 5301. (If no optional port is set, the default port 53 is used) +Extended Errors +^^^^^^^^^^^^^^^ +DNS messages can include extended error codes and text in the EDNS part of a reply. +If set, the Recursor will add the extended error code and text if resolving a name leads to an RPZ hit. +This information is then sent to the client, which can inspect the extended information for diagnosis and other purposes. +As an example consider + +.. code-block:: Lua + + rpzPrimary("192.0.2.4","policy.rpz", {extendedErrorCode = 15, extendedErrorExtra = "Blocked by policy"}) +Resolving a name blocked by this policy will produce ``dig`` output containing the following line: + + ; EDE: 15 (Blocked): 42 6c 6f 63 6b 65 64 20 62 79 20 70 6f 6c 69 63 79 ("Blocked by policy") + +Check :rfc:`8914` for other ``extendedErrorCodes``. + +RPZ Configuration Functions +--------------------------- .. function:: rpzFile(filename, settings) Load an RPZ from disk. diff --git a/pdns/recursordist/docs/lua-config/ztc.rst b/pdns/recursordist/docs/lua-config/ztc.rst new file mode 100644 index 000000000000..1408076ca517 --- /dev/null +++ b/pdns/recursordist/docs/lua-config/ztc.rst @@ -0,0 +1,75 @@ +.. _ztc: + +Zone to Cache +------------- + +Zone to Cache is a function to load a zone into the Recursor cache periodically, or every time the Lua configuration is loaded, at startup and whenever ``rec_control reload-lua-config`` is issued. +This allows the Recursor to have an always hot cache for these zones. +The zone content to cache can be retrieved via zone transfer (AXFR format) or read from a zone file retrieved via http, https or a local file. + +Example +^^^^^^^ +To load the root zone from Internic into the recursor once at startup and when the Lua config is reloaded: + +.. code-block:: Lua + + zoneToCache(".", "url", "https://www.internic.net/domain/root.zone", { refreshPeriod = 0 }) + +Configuration +^^^^^^^^^^^^^ +.. function:: zoneToCache(zone, method, source [, settings ]) + + .. versionadded:: 4.6.0 + + Load a zone and put it into the Recursor cache periodically. + + :param str zone: The name of the zone to load + :param str method: One of ``"axfr"``, ``"url"`` or ``"file"`` + :param str source: A string representing an IP address (when using the ``axfr`` method), URL (when using the ``url`` method) or path name (when using the ``file`` method) + :param table settings: A table of settings, see below + + +Zone to Cache settings +^^^^^^^^^^^^^^^^^^^^^^ + +These options can be set in the ``settings`` of :func:`zoneToCache`. + +timeout +~~~~~~~ +The maximum time (in seconds) a retrieval using the ``axfr`` or ``url`` method may take. +Default is 20 seconds. + +tsigname +~~~~~~~~ +The name of the TSIG key to authenticate to the server and validate the zone content with when using the ``axfr`` method. +When this is set, `tsigalgo`_ and `tsigsecret`_ must also be set. + +tsigalgo +~~~~~~~~ +The name of the TSIG algorithm (like 'hmac-md5') used. + +tsigsecret +~~~~~~~~~~ +Base64 encoded TSIG secret. + +refreshPeriod +~~~~~~~~~~~~~ +An integer describing the interval (in seconds) to wait between retrievals. +A value of zero means the retrieval is done once at startup and on Lua configuration reload. +By default, the refresh value is 86400 (24 hours). + +retryOnErrorPeriod +~~~~~~~~~~~~~~~~~~ +An integer describing the interval (in seconds) to wait before retrying a failed transfer. +By default 60 is used. + +maxReceivedMBytes +~~~~~~~~~~~~~~~~~ +The maximum size in megabytes of an update via the ``axfr`` or ``url`` methods, to prevent resource exhaustion. +The default value of 0 means no restriction. + +localAddress +~~~~~~~~~~~~ +The source IP address to use when transferring using the ``axfr`` or ``url`` methods. +When unset, :ref:`setting-query-local-address` is used. + diff --git a/pdns/recursordist/docs/lua-scripting/dq.rst b/pdns/recursordist/docs/lua-scripting/dq.rst index 9c2a3eddf800..f1becb4e45b3 100644 --- a/pdns/recursordist/docs/lua-scripting/dq.rst +++ b/pdns/recursordist/docs/lua-scripting/dq.rst @@ -33,10 +33,6 @@ The DNSQuestion object contains at least the following fields: :class:`DNSName` of the name this query is for. - .. attribute:: DNSQuestion.qname - - :class:`DNSName` of the name this query is for. - .. attribute:: DNSQuestion.qtype Type this query is for as an integer, can be compared against ``pdns.A``, ``pdns.AAAA``. diff --git a/pdns/recursordist/docs/lua-scripting/netmask.rst b/pdns/recursordist/docs/lua-scripting/netmask.rst index 3f2f8aa7ca8f..dae9137974f3 100644 --- a/pdns/recursordist/docs/lua-scripting/netmask.rst +++ b/pdns/recursordist/docs/lua-scripting/netmask.rst @@ -95,6 +95,7 @@ They can be matched against netmasks objects: Prefixing a mask with ``!`` excludes that mask from matching. .. function:: newNMG([masks]) -> NetMaskGroup + .. versionchanged:: 4.6.0 Added the optional ``masks`` parameter. diff --git a/pdns/recursordist/docs/manpages/rec_control.1.rst b/pdns/recursordist/docs/manpages/rec_control.1.rst index 42c1607498b2..eda01c07fb87 100644 --- a/pdns/recursordist/docs/manpages/rec_control.1.rst +++ b/pdns/recursordist/docs/manpages/rec_control.1.rst @@ -152,6 +152,13 @@ get-parameter *KEY* [*KEY*]... get-qtypelist Retrieves QType statistics. Queries from cache aren't being counted yet. +hash-password [*WORK-FACTOR*] + Asks for a password then returns the hashed and salted version, + to use as a webserver password or API key. This command does + not contact the recursor but does the hashing inside rec_control. + An optional scrypt work factor can be specified, in power of two. + The default is 1024. + help Shows a list of supported commands understood by the running :program:`pdns_recursor` @@ -211,6 +218,9 @@ set-max-packetcache-entries *NUM* set-minimum-ttl *NUM* Set minimum-ttl-override to *NUM*. +set-event-trace-enabled *NUM* + Set logging of event trace messages, 0 = disabled, 1 = protobuf, 2 = log file, 3 = both. + top-queries Shows the top-20 queries. Statistics are over the last 'stats-ringbuffer-entries' queries. diff --git a/pdns/recursordist/docs/metrics.rst b/pdns/recursordist/docs/metrics.rst index a3a83c338cf3..5223f78c8659 100644 --- a/pdns/recursordist/docs/metrics.rst +++ b/pdns/recursordist/docs/metrics.rst @@ -238,7 +238,8 @@ counts the number of queries to locally hosted authoritative zones (:ref:`settin cache-bytes ^^^^^^^^^^^ -size of the cache in bytes +size of the cache in bytes (disabled by default, see :ref:`setting-stats-rec-control-disabled-list`) +This metric is a rough estimate and takes a long time to compute, and is therefore not enabled in default outputs. cache-entries ^^^^^^^^^^^^^ @@ -284,28 +285,22 @@ cpu-steal Stolen time, which is the time spent by the whole system in other operating systems when running in a virtualized environment, in units of USER_HZ. -cumul-answers-x -^^^^^^^^^^^^^^^^^^ -.. versionadded:: 4.6 - -Cumulative counts of answer times in buckets less or equal than x microseconds. -These metrics include packet cache hits. -These metrics are useful for Prometheus and not listed other outputs by default. - -cumul-auth4-answers-x +cumul-authanswers-x ^^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 4.6 -Cumulative counts of answer times of authoritative servers over IPv4 in buckets less than x microseconds. -These metrics are useful for Prometheus and not listed other outputs by default. +Cumulative counts of answer times of authoritative servers in buckets less than x microseconds. +(disabled by default, see :ref:`setting-stats-rec-control-disabled-list`) +These metrics are useful for Prometheus and not listed in other outputs by default. -cumul-auth6-answers-x +cumul-clientanswers-x ^^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 4.6 -Cumulative counts of answer times of authoritative servers over IPv6 in buckets less than x microseconds. -These metrics are useful for Prometheus and not listed other outputs by default. +Cumulative counts of our answer times to clients in buckets less or equal than x microseconds. +These metrics include packet cache hits. +These metrics are useful for Prometheus and not listed in other outputs by default. dns64-prefix-answers ^^^^^^^^^^^^^^^^^^^^ @@ -475,12 +470,14 @@ ecs-v4-response-bits-* .. versionadded:: 4.2.0 number of responses received from authoritative servers with an IPv4 EDNS Client Subnet option we used, of this subnet size (1 to 32). +(disabled by default, see :ref:`setting-stats-rec-control-disabled-list`) ecs-v6-response-bits-* ^^^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 4.2.0 number of responses received from authoritative servers with an IPv6 EDNS Client Subnet option we used, of this subnet size (1 to 128). +(disabled by default, see :ref:`setting-stats-rec-control-disabled-list`) edns-ping-matches ^^^^^^^^^^^^^^^^^ @@ -494,6 +491,13 @@ failed-host-entries ^^^^^^^^^^^^^^^^^^^ number of servers that failed to resolve +.. _stat-fd-usage: + +fd-usage +^^^^^^^^ +Number of currently used file descriptors. +Currently, this metric is available on Linux and OpenBSD only. + ignored-packets ^^^^^^^^^^^^^^^ counts the number of non-query packets received on server sockets that should only get query packets @@ -576,7 +580,8 @@ questions dropped because over maximum concurrent query limit (since 3.2) packetcache-bytes ^^^^^^^^^^^^^^^^^ -size of the packet cache in bytes (since 3.3.1) +size of the packet cache in bytes (since 3.3.1) (disabled by default, see :ref:`setting-stats-rec-control-disabled-list`) +This metric is a rough estimate and takes a long time to compute, and is therefore not enabled in default outputs. packetcache-entries ^^^^^^^^^^^^^^^^^^^ @@ -663,7 +668,10 @@ number of contended record cache lock acquisitions resource-limits ^^^^^^^^^^^^^^^ -counts number of queries that could not be performed because of resource limits +Counts the number of queries that could not be performed because of resource limits. +This counter is increased when Recursor encounters a network issue that does not seem to be caused by the remote end. +For example when it runs out of file descriptors (monitor :ref:`stat-fd-usage`) or when there is no route to a +given IP address. security-status ^^^^^^^^^^^^^^^ @@ -755,6 +763,14 @@ unauthorized-udp ^^^^^^^^^^^^^^^^ number of UDP questions denied because of allow-from restrictions +source-disallowed-notify +^^^^^^^^^^^^^^^^^^^^^^^^ +number of NOTIFY operations denied because of allow-notify-from restrictions + +zone-disallowed-notify +^^^^^^^^^^^^^^^^^^^^^^ +number of NOTIFY operations denied because of allow-notify-for restrictions + unexpected-packets ^^^^^^^^^^^^^^^^^^ number of answers from remote servers that were unexpected (might point to spoofing) diff --git a/pdns/recursordist/docs/performance.rst b/pdns/recursordist/docs/performance.rst index b8b9ca663b41..f4f9f023630d 100644 --- a/pdns/recursordist/docs/performance.rst +++ b/pdns/recursordist/docs/performance.rst @@ -134,6 +134,30 @@ If you operate an anycast pool of machines, make them share the TCP Fast Open Ke To determine a good value for the :ref:`setting-tcp-fast-open` setting, watch the ``TCPFastOpenListenOverflow`` metric. If this value increases often, the value might be too low for your traffic, but note that increasing it will use kernel resources. +Running with a local root zone +------------------------------ +Running with a local root zone as described in :rfc:`8806` can help reduce traffic to the root servers and reduce response times for clients. +Since 4.6.0 PowerDNS Recursor supports two ways of doing this. + +Running a local Authoritative Server for the root zone + +- The first method is to have a local Authoritative Server that has a copy of the root zone and forward queries to it. + Setting up an PowerDNS Authoritative Server to serve a copy of the root zone looks like: + + pdnsutil create-secondary-zone . ip1 ip2 + + where ``ip1`` and ``ip2`` are servers willing to serve an AXFR for the root zone; :rfc:`8806` contains a list of candidates in appendix A. The Authoritative Server will periodically make sure its copy of the root zone is up-to-date. + The next step is to configure a forward zone to the IP ``ip`` of the Authoritative Server in the settings file or the Recursor: + + forward-zones=.=ip + + The Recursor will use the Authoritative Server to ask questions about the root zone, but if it learns about delegations still follow those. + Multiple Recursors can use this Authoritative Server. + +- The second method is to cache the root zone as described in :ref:`ztc`. + Here each Recursor will download and fill its cache with the contents of the root zone. + Depending on the ``timeout`` parameter, this will be done once or periodically. + Refer to :ref:`ztc` for details. Recursor Caches --------------- @@ -169,3 +193,108 @@ Measuring performance --------------------- The PowerDNS Recursor exposes many :doc:`metrics ` that can be graphed and monitored. + +Event Tracing +------------- +Event tracing is an experimental feature introduced in version 4.6.0 that allows following the internals of processing queries in more detail. + +In certain spots in the resolving process event records are created that contain an identification of the event, a timestamp, potentially a value and an indication if this was the start or the end of an event. This is relevant for events that describe stages in the resolving process. + +At this point in time event logs of queries can be exported using a protobuf log or they can be written to the log file. + +Note that this is an experimental feature that will change in upcoming releases. + +Currently, an event protobuf message has the following definition: + +.. code-block:: protobuf + + enum EventType { + // Range 0..99: Generic events + CustomEvent = 0; // A custom event + ReqRecv = 1; // A request was received + PCacheCheck = 2; // A packet cache check was initiated or completed; value: bool cacheHit + AnswerSent = 3; // An answer was sent to the client + + // Range 100: Recursor events + SyncRes = 100; // Recursor Syncres main function has started or completed; value: int rcode + LuaGetTag = 101; // Events below mark start or end of Lua hook calls; value: return value of hook + LuaGetTagFFI = 102; + LuaIPFilter = 103; + LuaPreRPZ = 104; + LuaPreResolve = 105; + LuaPreOutQuery = 106; + LuaPostResolve = 107; + LuaNoData = 108; + LuaNXDomain = 109; + } + +.. code-block:: protobuf + + message Event { + required uint64 ts = 1; + required EventType event = 2; + required bool start = 3; + optional bool boolVal = 4; + optional int64 intVal = 5; + optional string stringVal = 6; + optional bytes bytesVal = 7; + optional string custom = 8; + } + repeated Event trace = 23; + +Event traces can be enabled by either setting :ref:`setting-event-trace-enabled` or by using the :doc:`rec_control ` subcommand ``set-event-trace-enabled``. + +An example of a trace (timestamps are relative in nanoseconds) as shown in the logfile: + +.. code-block:: C + + - ReqRecv(70); + - PCacheCheck(411964); + - PCacheCheck(416783,0,done); + - SyncRes(441811); + - SyncRes(337233971,0,done); + -AnswerSent(337266453) + +The packet cache check event has two events. +The first signals the start of packet cache lookup, and the second the completion of the packet cache lookup with result 0 (not found). +The SynRec event also has two entries. The value (0) is the return value of the SyncRes function. + +An example of a trace with a packet cache hit): + +.. code-block:: C + + - ReqRecv(60); + - PCacheCheck(22913); + - PCacheCheck(113255,1,done); + - AnswerSent(117493) + +Here it can be seen that packet cache returns 1 (found). + +An example where various Lua related events can be seen: + +.. code-block:: C + + ReqRecv(150); + PCacheCheck(26912); + PCacheCheck(51308,0,done); + LuaIPFilter(56868); + LuaIPFilter(57149,0,done); + LuaPreRPZ(82728); + LuaPreRPZ(82918,0,done); + LuaPreResolve(83479); + LuaPreResolve(210621,0,done); + SyncRes(217424); + LuaPreOutQuery(292868); + LuaPreOutQuery(292938,0,done); + LuaPreOutQuery(24702079); + LuaPreOutQuery(24702349,0,done); + LuaPreOutQuery(43055303); + LuaPreOutQuery(43055634,0,done); + SyncRes(80470320,0,done); + LuaPostResolve(80476592); + LuaPostResolve(80476772,0,done); + AnswerSent(80500247) + +There is no packet cache hit, so SyncRes is called which does a couple of outgoing queries. + + diff --git a/pdns/recursordist/docs/requirements.txt b/pdns/recursordist/docs/requirements.txt index cd61585a5325..573799b8527d 100644 --- a/pdns/recursordist/docs/requirements.txt +++ b/pdns/recursordist/docs/requirements.txt @@ -5,4 +5,5 @@ git+https://github.com/pieterlexis/sphinx-changelog@render-tags guzzle_sphinx_theme sphinxcontrib.httpdomain sphinxcontrib-fulltoc -docutils!=0.15 +docutils!=0.15,<0.18 +jinja2<3.1.0 diff --git a/pdns/recursordist/docs/settings.rst b/pdns/recursordist/docs/settings.rst index 75c7a15122a4..0bb4de78d79d 100644 --- a/pdns/recursordist/docs/settings.rst +++ b/pdns/recursordist/docs/settings.rst @@ -58,6 +58,69 @@ Note that specifying an IP address without a netmask uses an implicit netmask of Like `allow-from`_, except reading from file. Overrides the `allow-from`_ setting. To use this feature, supply one netmask per line, with optional comments preceded by a "#". +.. _setting-allow-notify-for: + +``allow-notify-for`` +--------------------- +.. versionadded:: 4.6.0 + +- Comma separated list of domain-names +- Default: (empty) + +Domain names specified in this list are used to permit incoming +NOTIFY operations to wipe any cache entries that match the domain +name. If this list is empty, all NOTIFY operations will be ignored. + +.. _setting-allow-notify-for-file: + +``allow-notify-for-file`` +------------------------- +.. versionadded:: 4.6.0 + +- Path + +Like `allow-notify-for`_, except reading from file. To use this +feature, supply one domain name per line, with optional comments +preceded by a "#". + +.. _setting-allow-notify-from: + +``allow-notify-from`` +--------------------- +.. versionadded:: 4.6.0 + +- IP addresses or netmasks, separated by commas +- Default: unset + +Netmasks (both IPv4 and IPv6) that are allowed to issue NOTIFY operations +to the server. NOTIFY operations from IP addresses not listed here are +ignored and do not get an answer. + +When the Proxy Protocol is enabled (see `proxy-protocol-from`_), the +recursor will check the address of the client IP advertised in the +Proxy Protocol header instead of the one of the proxy. + +Note that specifying an IP address without a netmask uses an implicit +netmask of /32 or /128. + +NOTIFY operations received from a client listed in one of these netmasks +will be accepted and used to wipe any cache entries whose zones match +the zone specified in the NOTIFY operation, but only if that zone (or +one of its parents) is included in `allow-notify-for`_, +`allow-notify-for-file`_, or `forward-zones-file_` with a '^' prefix. + +.. _setting-allow-notify-from-file: + +``allow-notify-from-file`` +-------------------------- +.. versionadded:: 4.6.0 + +- Path + +Like `allow-notify-from`_, except reading from file. To use this +feature, supply one netmask per line, with optional comments preceded +by a "#". + .. _setting-any-to-tcp: ``any-to-tcp`` @@ -95,11 +158,13 @@ Directory where the REST API stores its configuration and zones. ``api-key`` ----------- .. versionadded:: 4.0.0 +.. versionchanged:: 4.6.0 + This setting now accepts a hashed and salted version. - String - Default: unset -Static pre-shared authentication key for access to the REST API. +Static pre-shared authentication key for access to the REST API. Since 4.6.0 the key can be hashed and salted using ``rec_control hash-password`` instead of being stored in the configuration in plaintext, but the plaintext version is still supported. .. _setting-api-readonly: @@ -208,7 +273,8 @@ See :doc:`metrics`. - Path to a Directory If set, chroot to this directory for more security. -See :doc:`security` +This is not recommended; instead, we recommend containing PowerDNS using operating system features. +We ship systemd unit files with our packages to make this easy. Make sure that ``/dev/log`` is available from within the chroot. Logging will silently fail over time otherwise (on logrotate). @@ -386,7 +452,7 @@ If `pdns-distributes-queries`_ is set, spawn this number of distributor threads handle incoming queries and distribute them to other threads based on a hash of the query, to maximize the cache hit ratio. -.. _settings-dot-to-auth-names: +.. _setting-dot-to-auth-names: ``dot-to-auth-names`` --------------------- @@ -398,7 +464,7 @@ ratio. Force DoT to the listed authoritative nameservers. For this to work, DoT support has to be compiled in. Currently, the certificate is not checked for validity in any way. -.. _settings-dot-to-port-853: +.. _setting-dot-to-port-853: ``dot-to-port-853`` ------------------- @@ -704,6 +770,18 @@ Change to ``/dev/random`` if PowerDNS should block waiting for enough entropy to The path to the /etc/hosts file, or equivalent. This file can be used to serve data authoritatively using `export-etc-hosts`_. +.. _setting-event-trace-enabled: + +``event-trace-enabled`` +----------------------- +.. versionadded:: 4.6.0 + +- Integer +- Default: 0 + +Enable the recording and logging of ref:`event traces`. This is an experimental feature subject to change. +Possible values are 0: (disabled), 1 (add information to protobuf logging messages) and 2 (write to log) and 3 (both). + .. _setting-export-etc-hosts: ``export-etc-hosts`` @@ -771,8 +849,9 @@ Same as `forward-zones`_, parsed from a file. Only 1 zone is allowed per line, s example.org=203.0.113.210, 192.0.2.4:5300 -Zones prefixed with a '+' are forwarded with the recursion-desired bit set, for which see `forward-zones-recurse`_. -Default behaviour without '+' is as with `forward-zones`_. +Zones prefixed with a '+' are treated as with +`forward-zones-recurse`_. Default behaviour without '+' is as with +`forward-zones`_. .. versionchanged:: 4.0.0 @@ -780,6 +859,11 @@ Default behaviour without '+' is as with `forward-zones`_. The DNSSEC notes from `forward-zones`_ apply here as well. +.. versionchanged:: 4.6.0 + +Zones prefixed with a '^' are added to the `allow-notify-for`_ +list. Both prefix characters can be used if desired, in any order. + .. _setting-forward-zones-recurse: ``forward-zones-recurse`` @@ -788,7 +872,7 @@ The DNSSEC notes from `forward-zones`_ apply here as well. Like regular `forward-zones`_, but forwarded queries have the 'recursion desired' bit set to 1, meaning that this setting is intended to forward queries to other recursive servers. -The DNSSEC notes from `forward-zones`_ apply here as well. +See `forward-zones`_ for additional options (such as supplying multiple recursive servers) and an important note about DNSSEC. .. _setting-gettag-needs-edns-options: @@ -806,8 +890,16 @@ If set, EDNS options in incoming queries are extracted and passed to the :func:` ``hint-file`` ------------- - Path +- Default: empty -If set, the root-hints are read from this file. If unset, default root hints are used. +.. versionchanged:: 4.7.0 + + Introduced the value ``no`` to disable root-hints processing. + +If set, the root-hints are read from this file. If empty, the default built-in root hints are used. + +In some special cases, processing the root hints is not needed, for example when forwarding all queries to another recursor. +For these special cases, it is possible to disable the processing of root hints by setting the value to ``no``. .. _setting-ignore-unknown-settings: @@ -1030,6 +1122,19 @@ Maximum number of incoming requests handled concurrently per tcp connection. This number must be larger than 0 and smaller than 65536 and also smaller than `max-mthreads`. +.. _setting-max-include-depth: + +``max-include-depth`` +---------------------- + +.. versionadded:: 4.6.0 + +- Integer +- Default: 20 + +Maximum number of nested ``$INCLUDE`` directives while processing a zone file. +Zero mean no ``$INCLUDE`` directives will be accepted. + .. _setting-max-generate-steps: ``max-generate-steps`` @@ -1411,7 +1516,8 @@ Maximum number of seconds to cache an item in the packet cache, no matter what t - Integer - Default: 60 -Maximum number of seconds to cache a 'server failure' answer in the packet cache. +Maximum number of seconds to cache an answer indicating a failure to resolve in the packet cache. +Before version 4.6.0 only ``ServFail`` answers were considered as such. Starting with 4.6.0, all responses with a code other than ``NoError`` and ``NXDomain``, or without records in the answer and authority sections, are considered as a failure to resolve. .. versionchanged:: 4.0.0 @@ -1455,6 +1561,8 @@ Queries that are not prefixed with such a header will not be accepted from clien Note that once a Proxy Protocol header has been received, the source address from the proxy header instead of the address of the proxy will be checked against the `allow-from`_ ACL. +The dnsdist docs have `more information about the PROXY protocol `_. + .. _setting-proxy-protocol-maximum-size: ``proxy-protocol-maximum-size`` @@ -1862,6 +1970,17 @@ Can be read out using ``rec_control top-remotes``. A list of comma-separated statistic names, that are prevented from being exported via SNMP, for performance reasons. +.. _setting-structured-logging: + +``structured-logging`` +---------------------- +.. versionadded:: 4.6.0 + +- Boolean +- Default: yes + +Prefer structured logging when both an old style and a structured log messages is available. + .. _setting-tcp-fast-open: ``tcp-fast-open`` @@ -1885,6 +2004,51 @@ The numerical value supplied is used as the queue size, 0 meaning disabled. See Enable TCP Fast Open Connect support, if available, on the outgoing connections to authoritative servers. See :ref:`tcp-fast-open-support`. +.. _setting-tcp-out-max-idle-ms: + +``tcp-out-max-idle-ms`` +----------------------- +.. versionadded:: 4.6.0 + +- Integer +- Default : 10000 + +Time outgoing TCP/DoT connections are left idle in milliseconds or 0 if no limit. After having been idle for this time, the connection is eligible for closing. + +.. _setting-tcp-out-max-idle-per-auth: + +``tcp-out-max-idle-per-auth`` +----------------------------- +.. versionadded:: 4.6.0 + +- Integer +- Default : 10 + +Maximum number of idle outgoing TCP/DoT connections to a specific IP per thread, 0 means do not keep idle connections open. + +.. _setting-tcp-out-max-queries: + +``tcp-out-max-queries`` +----------------------- +- Integer +- Default : 0 + +Maximum total number of queries per outgoing TCP/DoT connection, 0 means no limit. After this number of queries, the connection is +closed and a new one will be created if needed. + +.. versionadded:: 4.6.0 + +.. _setting-tcp-out-max-idle-per-thread: + +``tcp-out-max-idle-per-thread`` +------------------------------- +.. versionadded:: 4.6.0 + +- Integer +- Default : 0 + +Maximum number of idle outgoing TCP/DoT connections per thread, 0 means do not keep idle connections open. + .. _setting-threads: ``threads`` @@ -2108,6 +2272,18 @@ These IPs and subnets are allowed to access the webserver. Note that specifying an IP address without a netmask uses an implicit netmask of /32 or /128. +.. _setting-webserver-hash-plaintext-credentials: + +``webserver-hash-plaintext-credentials`` +---------------------------------------- +..versionadded:: 4.6.0 + +- Boolean +- Default: no + +Whether passwords and API keys supplied in the configuration as plaintext should be hashed during startup, to prevent the plaintext versions from staying in memory. Doing so increases significantly the cost of verifying credentials and is thus disabled by default. +Note that this option only applies to credentials stored in the configuration as plaintext, but hashed credentials are supported without enabling this option. + .. _setting-webserver-loglevel: ``webserver-loglevel`` @@ -2153,10 +2329,13 @@ The value between the hooks is a UUID that is generated for each request. This c ``webserver-password`` ---------------------- +.. versionchanged:: 4.6.0 + This setting now accepts a hashed and salted version. + - String - Default: unset -Password required to access the webserver. +Password required to access the webserver. Since 4.6.0 the password can be hashed and salted using ``rec_control hash-password`` instead of being present in the configuration in plaintext, but the plaintext version is still supported. .. _setting-webserver-port: diff --git a/pdns/recursordist/docs/upgrade.rst b/pdns/recursordist/docs/upgrade.rst index 64f0a6064e05..1d8475e6b03a 100644 --- a/pdns/recursordist/docs/upgrade.rst +++ b/pdns/recursordist/docs/upgrade.rst @@ -4,8 +4,34 @@ Upgrade Guide Before upgrading, it is advised to read the :doc:`changelog/index`. When upgrading several versions, please read **all** notes applying to the upgrade. -4.5.x to 4.6.0 or master ------------------------- +4.6.x to master +--------------- + +Deprecated and changed settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- The :ref:`setting-hint-file` gained a special value ``no`` to indicate that no hint file should not processed. + +4.5.x to 4.6.0 +-------------- + +Offensive language +^^^^^^^^^^^^^^^^^^ +Using the settings mentioned in :ref:`upgrade-offensive` now generates a warning. Please start using the new names. + +New settings +^^^^^^^^^^^^ +- The :ref:`setting-dot-to-auth-names` setting to list nameservers that should be contacted over DoT has been introduced. +- The :ref:`setting-dot-to-port-853` setting to specify that nameservers or forwarders using port 853 should be contacted over DoT has been introduced. +- The :ref:`setting-ignore-unknown-settings` setting has been introduced to make it easier to switch between recursor versions supporting different settings. +- The :ref:`setting-webserver-hash-plaintext-credentials` has been introduced to avoid keeping cleartext sensitive information in memory. +- The :ref:`setting-tcp-out-max-idle-ms`, :ref:`setting-tcp-out-max-idle-per-auth`, :ref:`setting-tcp-out-max-queries` and :ref:`setting-tcp-out-max-idle-per-thread` settings have been introduced to control the new TCP/DoT outgoing connections pooling. This mechanism keeps connections to authoritative servers or forwarders open for later re-use. +- The :ref:`setting-structured-logging` setting has been introduced to prefer structured logging (the default) when both an old style and a structured log messages is available. +- The :ref:`setting-max-include-depth` setting has been introduced to limit the number of nested ``$include`` directives while processing a zone file. + +Deprecated and changed settings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- The :ref:`setting-api-key` and :ref:`setting-webserver-password` settings now accept a hashed and salted version (if the support is available in the openssl library used). + 4.5.1 to 4.5.2 -------------- @@ -17,6 +43,8 @@ Deprecated and changed settings 4.4.x to 4.5.1 -------------- +.. _upgrade-offensive: + Offensive language ^^^^^^^^^^^^^^^^^^ Synonyms for various settings names containing ``master``, ``slave``, diff --git a/pdns/recursordist/logging.cc b/pdns/recursordist/logging.cc index ace669f0ed56..6196ba3b8f7a 100644 --- a/pdns/recursordist/logging.cc +++ b/pdns/recursordist/logging.cc @@ -95,7 +95,7 @@ void Logger::error(const std::string& err, const std::string& msg) const std::shared_ptr Logger::v(size_t level) const { - auto res = std::make_shared(getptr(), boost::none, getVerbosity(), level + _level, _callback); + auto res = std::make_shared(getptr(), _name, getVerbosity(), level + _level, _callback); return res; } @@ -177,3 +177,4 @@ Logger::~Logger() }; std::shared_ptr g_slog{nullptr}; +bool g_slogStructured = true; diff --git a/pdns/recursordist/logging.hh b/pdns/recursordist/logging.hh index fe5ad1f4fa9f..fb03332d3ad3 100644 --- a/pdns/recursordist/logging.hh +++ b/pdns/recursordist/logging.hh @@ -116,3 +116,22 @@ private: } extern std::shared_ptr g_slog; + +// Prefer structured logging? +extern bool g_slogStructured; + +// A helper macro to switch between old-style logging and new-style (structured logging) +// A typical use: +// +// SLOG(g_log<error("No such file", "Unable to parse configuration file", "config_file", Logging::Loggable(configname)); +// +#define SLOG(oldStyle, slogCall) \ + do { \ + if (g_slogStructured) { \ + slogCall; \ + } \ + else { \ + oldStyle; \ + } \ + } while (0); diff --git a/pdns/recursordist/m4/libcurl.m4 b/pdns/recursordist/m4/libcurl.m4 new file mode 120000 index 000000000000..91d8843a09fd --- /dev/null +++ b/pdns/recursordist/m4/libcurl.m4 @@ -0,0 +1 @@ +../../../m4/libcurl.m4 \ No newline at end of file diff --git a/pdns/recursordist/m4/pdns_check_libcurl.m4 b/pdns/recursordist/m4/pdns_check_libcurl.m4 new file mode 120000 index 000000000000..0e21701cec10 --- /dev/null +++ b/pdns/recursordist/m4/pdns_check_libcurl.m4 @@ -0,0 +1 @@ +../../../m4/pdns_check_libcurl.m4 \ No newline at end of file diff --git a/pdns/recursordist/minicurl.cc b/pdns/recursordist/minicurl.cc new file mode 120000 index 000000000000..3f48b8c54eea --- /dev/null +++ b/pdns/recursordist/minicurl.cc @@ -0,0 +1 @@ +../minicurl.cc \ No newline at end of file diff --git a/pdns/recursordist/minicurl.hh b/pdns/recursordist/minicurl.hh new file mode 120000 index 000000000000..bba06c379e70 --- /dev/null +++ b/pdns/recursordist/minicurl.hh @@ -0,0 +1 @@ +../minicurl.hh \ No newline at end of file diff --git a/pdns/recursordist/negcache.hh b/pdns/recursordist/negcache.hh index 8e2d9c7e8af4..c2a0fcbf96a3 100644 --- a/pdns/recursordist/negcache.hh +++ b/pdns/recursordist/negcache.hh @@ -29,6 +29,7 @@ #include "dnsname.hh" #include "dns.hh" #include "lock.hh" +#include "stat_t.hh" #include "validate.hh" using namespace ::boost::multi_index; @@ -113,7 +114,7 @@ private: uint64_t d_acquired_count{0}; void invalidate() {} }; - std::atomic d_entriesCount{0}; + pdns::stat_t d_entriesCount{0}; LockGuardedTryHolder lock() { diff --git a/pdns/recursordist/pdns-recursor.service.in b/pdns/recursordist/pdns-recursor.service.in index 8707d567a163..886b90655da0 100644 --- a/pdns/recursordist/pdns-recursor.service.in +++ b/pdns/recursordist/pdns-recursor.service.in @@ -14,6 +14,7 @@ Type=notify Restart=on-failure StartLimitInterval=0 RuntimeDirectory=pdns-recursor +SyslogIdentifier=pdns-recursor # Tuning LimitNOFILE=16384 diff --git a/pdns/recursordist/rec-eventtrace.cc b/pdns/recursordist/rec-eventtrace.cc new file mode 100644 index 000000000000..6060dbd5f767 --- /dev/null +++ b/pdns/recursordist/rec-eventtrace.cc @@ -0,0 +1,43 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#include "rec-eventtrace.hh" + +#define NameEntry(M) \ + { \ + M, #M \ + } + +const std::unordered_map RecEventTrace::s_eventNames = { + NameEntry(CustomEvent), + NameEntry(ReqRecv), + NameEntry(PCacheCheck), + NameEntry(AnswerSent), + NameEntry(SyncRes), + NameEntry(LuaGetTag), + NameEntry(LuaGetTagFFI), + NameEntry(LuaIPFilter), + NameEntry(LuaPreRPZ), + NameEntry(LuaPreResolve), + NameEntry(LuaPreOutQuery), + NameEntry(LuaPostResolve), + NameEntry(LuaNoData), + NameEntry(LuaNXDomain)}; diff --git a/pdns/recursordist/rec-eventtrace.hh b/pdns/recursordist/rec-eventtrace.hh new file mode 100644 index 000000000000..4c64ac45c3e9 --- /dev/null +++ b/pdns/recursordist/rec-eventtrace.hh @@ -0,0 +1,271 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include "namespaces.hh" +#include "misc.hh" +#include "noinitvector.hh" + +#include +#include +#include +#include + +class RecEventTrace +{ +public: + enum EventType : uint8_t + { + // Keep in-syc with dnsmessagge.proto! + // Don't forget to add a new entry to the table in the .cc file! + // Generic events + CustomEvent = 0, + ReqRecv = 1, + PCacheCheck = 2, + AnswerSent = 3, + + // Recursor specific events + SyncRes = 100, + LuaGetTag = 101, + LuaGetTagFFI = 102, + LuaIPFilter = 103, + LuaPreRPZ = 104, + LuaPreResolve = 105, + LuaPreOutQuery = 106, + LuaPostResolve = 107, + LuaNoData = 108, + LuaNXDomain = 109, + }; + + static const std::unordered_map s_eventNames; + + RecEventTrace() + { + reset(); + } + + RecEventTrace(const RecEventTrace& old) : + d_events(old.d_events), + d_base(old.d_base), + d_status(old.d_status) + { + // An RecEventTrace object can be copied, but the original will be marked invalid. + // This is do detect (very likely) unintended modifications to the original after + // the ownership changed. + old.d_status = Invalid; + } + + RecEventTrace(RecEventTrace&& old) : + d_events(std::move(old.d_events)), + d_base(old.d_base), + d_status(old.d_status) + { + // An RecEventTrace object can be moved, but the original will be marked invalid. + // This is do detect (very likely) unintended modifications to the original after + // the ownership changed. + old.d_status = Invalid; + } + + RecEventTrace& operator=(const RecEventTrace& old) = delete; + RecEventTrace& operator=(RecEventTrace&& old) + { + d_events = std::move(old.d_events); + d_base = old.d_base; + d_status = old.d_status; + old.d_status = Invalid; + return *this; + } + + // We distinguish between strings and byte arrays. Does not matter in C++, but in Go, Java etc it does + typedef std::variant Value_t; + + static std::string toString(const EventType v) + { + return s_eventNames.at(v); + } + + static std::string toString(const Value_t& v) + { + if (std::holds_alternative(v)) { + return ""; + } + else if (std::holds_alternative(v)) { + return std::to_string(std::get(v)); + } + else if (std::holds_alternative(v)) { + return std::to_string(std::get(v)); + } + else if (std::holds_alternative(v)) { + return std::get(v); + } + else if (std::holds_alternative(v)) { + const PacketBuffer& p = std::get(v); + return makeHexDump(std::string(reinterpret_cast(p.data()), p.size())); + } + return "?"; + } + + struct Entry + { + Entry(Value_t&& v, EventType e, bool start, int64_t ts) : + d_value(std::move(v)), d_ts(ts), d_event(e), d_start(start) + { + } + Entry(Value_t&& v, const std::string& custom, bool start, int64_t ts) : + d_value(std::move(v)), d_custom(custom), d_ts(ts), d_event(CustomEvent), d_start(start) + { + } + Value_t d_value; + std::string d_custom; + int64_t d_ts; + EventType d_event; + bool d_start; + + std::string toString() const + { + std::string v = RecEventTrace::toString(d_value); + if (!v.empty()) { + v = "," + v; + } + std::string name = RecEventTrace::toString(d_event); + if (d_event == EventType::CustomEvent) { + name += ":" + d_custom; + } + + return name + "(" + std::to_string(d_ts) + v + (d_start ? ")" : ",done)"); + } + }; + + void setEnabled(bool flag) + { + assert(d_status != Invalid); + d_status = flag ? Enabled : Disabled; + } + + bool enabled() const + { + return d_status == Enabled; + } + + template + void add(E e, Value_t&& v, bool start, int64_t stamp = 0) + { + assert(d_status != Invalid); + if (d_status == Disabled) { + return; + } + if (stamp == 0) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + stamp = ts.tv_nsec + ts.tv_sec * 1000000000; + } + if (stamp < d_base) { + // If we get a ts before d_base, we adjust d_base and the existing events + // This is possble if we add a kernel provided packet timestamp in the future + // (Though it seems those timestamps do not use CLOCK_MONOTONIC...) + const int64_t adj = d_base - stamp; + for (auto& i : d_events) { + i.d_ts += adj; + } + // and move to the new base + d_base = stamp; + } + stamp -= d_base; + d_events.emplace_back(std::move(v), e, start, stamp); + } + + template + void add(E e) + { + add(e, Value_t(std::nullopt), true); + } + + // We store uint32 in an int64_t + template + void add(E e, uint32_t v, bool start) + { + add(e, static_cast(v), start); + } + // We store int32 in an int64_t + template + void add(E e, int32_t v, bool start) + { + add(e, static_cast(v), start); + } + + template + void add(E e, T v, bool start) + { + add(e, Value_t(v), start); + } + + void clear() + { + d_events.clear(); + reset(); + } + + void reset() + { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + d_base = ts.tv_nsec + ts.tv_sec * 1000000000; + d_status = Disabled; + } + + std::string toString() const + { + assert(d_status != Invalid); + if (d_status == Disabled) { + return "Disabled\n"; + } + std::string ret = "eventTrace ["; + bool first = true; + for (const auto& e : d_events) { + if (first) { + first = false; + } + else { + ret += "; "; + } + ret += e.toString(); + } + ret += ']'; + return ret; + } + + const std::vector& getEvents() const + { + return d_events; + } + +private: + std::vector d_events; + int64_t d_base; + enum Status : uint8_t + { + Disabled, + Invalid, + Enabled + }; + mutable Status d_status{Disabled}; +}; diff --git a/pdns/recursordist/rec-protozero.cc b/pdns/recursordist/rec-protozero.cc index 86f366688585..8a698093d36a 100644 --- a/pdns/recursordist/rec-protozero.cc +++ b/pdns/recursordist/rec-protozero.cc @@ -22,6 +22,7 @@ #include "dnsrecords.hh" #include "rec-protozero.hh" +#include void pdns::ProtoZero::RecMessage::addRR(const DNSRecord& record, const std::set& exportTypes, bool udr) { @@ -109,3 +110,33 @@ void pdns::ProtoZero::RecMessage::clearUDR(std::string& str) } } #endif + +void pdns::ProtoZero::RecMessage::addEvents(const RecEventTrace& trace) +{ + for (const auto& t : trace.getEvents()) { + protozero::pbf_writer pbf_trace{d_message, static_cast(Field::trace)}; + pbf_trace.add_int64(static_cast(Event::ts), t.d_ts); + pbf_trace.add_uint32(static_cast(Event::event), t.d_event); + pbf_trace.add_bool(static_cast(Event::start), t.d_start); + + const auto& v = t.d_value; + if (std::holds_alternative(v)) { + } + else if (std::holds_alternative(v)) { + pbf_trace.add_bool(static_cast(Event::boolVal), std::get(v)); + } + else if (std::holds_alternative(v)) { + pbf_trace.add_int64(static_cast(Event::intVal), std::get(v)); + } + else if (std::holds_alternative(v)) { + pbf_trace.add_string(static_cast(Event::stringVal), std::get(v)); + } + else if (std::holds_alternative(v)) { + const PacketBuffer& p = std::get(v); + pbf_trace.add_bytes(static_cast(Event::bytesVal), reinterpret_cast(p.data()), p.size()); + } + if (!t.d_custom.empty()) { + pbf_trace.add_string(static_cast(Event::custom), t.d_custom); + } + } +} diff --git a/pdns/recursordist/rec-protozero.hh b/pdns/recursordist/rec-protozero.hh index 20f511168381..6502fd71a7ab 100644 --- a/pdns/recursordist/rec-protozero.hh +++ b/pdns/recursordist/rec-protozero.hh @@ -24,6 +24,7 @@ #include "protozero.hh" #include "filterpo.hh" +#include "rec-eventtrace.hh" #include "validate.hh" namespace pdns @@ -99,6 +100,8 @@ namespace ProtoZero return std::move(d_msgbuf); } + void addEvents(const RecEventTrace& trace); + // DNSResponse related fields below void addRR(const DNSRecord& record, const std::set& exportTypes, bool udr); diff --git a/pdns/recursordist/rec-taskqueue.cc b/pdns/recursordist/rec-taskqueue.cc index 2705c59b854d..a737de2e24d6 100644 --- a/pdns/recursordist/rec-taskqueue.cc +++ b/pdns/recursordist/rec-taskqueue.cc @@ -76,7 +76,16 @@ void runTaskOnce(bool logErrors) void pushAlmostExpiredTask(const DNSName& qname, uint16_t qtype, time_t deadline) { ++s_almost_expired_tasks_pushed; - t_taskQueue.push({qname, qtype, deadline, true, resolve}); + switch (qtype) { + // Internal types + case QType::ENT: + case QType::ADDR: + case QType::ALIAS: + case QType::LUA: + return; + } + pdns::ResolveTask task{qname, qtype, deadline, true, resolve}; + t_taskQueue.push(std::move(task)); } uint64_t getTaskPushes() diff --git a/pdns/recursordist/rec-tcpout.cc b/pdns/recursordist/rec-tcpout.cc new file mode 100644 index 000000000000..8854ce12e074 --- /dev/null +++ b/pdns/recursordist/rec-tcpout.cc @@ -0,0 +1,88 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "rec-tcpout.hh" + +// This line from /usr/include/openssl/ssl2.h: # define CERT char +// throws dnsrecords.hh off the rails. +#undef CERT + +#include "syncres.hh" + +timeval TCPOutConnectionManager::s_maxIdleTime; +size_t TCPOutConnectionManager::s_maxQueries; +size_t TCPOutConnectionManager::s_maxIdlePerAuth; +size_t TCPOutConnectionManager::s_maxIdlePerThread; + +void TCPOutConnectionManager::cleanup(const struct timeval& now) +{ + if (s_maxIdleTime.tv_sec == 0 && s_maxIdleTime.tv_usec == 0) { + // no maximum idle time + return; + } + + for (auto it = d_idle_connections.begin(); it != d_idle_connections.end();) { + timeval idle = now - it->second.d_last_used; + if (s_maxIdleTime < idle) { + it = d_idle_connections.erase(it); + } + else { + ++it; + } + } +} + +void TCPOutConnectionManager::store(const struct timeval& now, const ComboAddress& ip, Connection&& connection) +{ + ++connection.d_numqueries; + if (s_maxQueries > 0 && connection.d_numqueries > s_maxQueries) { + return; + } + + if (d_idle_connections.size() >= s_maxIdlePerThread || d_idle_connections.count(ip) >= s_maxIdlePerAuth) { + cleanup(now); + } + + if (d_idle_connections.size() >= s_maxIdlePerThread) { + return; + } + if (d_idle_connections.count(ip) >= s_maxIdlePerAuth) { + return; + } + + gettimeofday(&connection.d_last_used, nullptr); + d_idle_connections.emplace(ip, std::move(connection)); +} + +TCPOutConnectionManager::Connection TCPOutConnectionManager::get(const ComboAddress& ip) +{ + if (d_idle_connections.count(ip) > 0) { + auto h = d_idle_connections.extract(ip); + return h.mapped(); + } + return Connection{}; +} + +uint64_t getCurrentIdleTCPConnections() +{ + return broadcastAccFunction([] { return t_tcp_manager.getSize(); }); +} diff --git a/pdns/recursordist/rec-tcpout.hh b/pdns/recursordist/rec-tcpout.hh new file mode 100644 index 000000000000..9c2cfa4d5167 --- /dev/null +++ b/pdns/recursordist/rec-tcpout.hh @@ -0,0 +1,75 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "iputils.hh" +#include "tcpiohandler.hh" + +class TCPOutConnectionManager +{ +public: + // Max idle time for a connection, 0 is no timeout + static struct timeval s_maxIdleTime; + // Per thread maximum of idle connections for a specific destination, 0 means no idle connections will be kept open + static size_t s_maxIdlePerAuth; + // Max total number of queries to handle per connection, 0 is no max + static size_t s_maxQueries; + // Per thread max # of idle connections, 0 means no idle connections will be kept open + static size_t s_maxIdlePerThread; + + struct Connection + { + std::string toString() const + { + if (d_handler) { + return std::to_string(d_handler->getDescriptor()) + ' ' + std::to_string(d_handler.use_count()); + } + return ""; + } + + std::shared_ptr d_handler; + timeval d_last_used{0, 0}; + size_t d_numqueries{0}; + }; + + void store(const struct timeval& now, const ComboAddress& ip, Connection&& connection); + Connection get(const ComboAddress& ip); + void cleanup(const struct timeval& now); + + size_t size() const + { + return d_idle_connections.size(); + } + uint64_t* getSize() const + { + return new uint64_t(size()); + } + +private: + // This does not take into account that we can have multiple connections with different hosts (via SNI) to the same IP. + // That is OK, since we are connecting by IP only at the moment. + std::multimap d_idle_connections; +}; + +extern thread_local TCPOutConnectionManager t_tcp_manager; +uint64_t getCurrentIdleTCPConnections(); diff --git a/pdns/recursordist/rec-zonetocache.cc b/pdns/recursordist/rec-zonetocache.cc new file mode 100644 index 000000000000..fa3894bb9c57 --- /dev/null +++ b/pdns/recursordist/rec-zonetocache.cc @@ -0,0 +1,304 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "rec-zonetocache.hh" + +#include "syncres.hh" +#include "zoneparser-tng.hh" +#include "query-local-address.hh" +#include "axfr-retriever.hh" +#include "validate-recursor.hh" +#include "logging.hh" +#include "threadname.hh" +#include "rec-lua-conf.hh" + +#ifdef HAVE_LIBCURL +#include "minicurl.hh" +#endif + +#include + +struct ZoneData +{ + ZoneData(shared_ptr& log, const std::string& zone) : + d_log(log), + d_zone(zone), + d_now(time(nullptr)) {} + + // Potentially the two fields below could be merged into a single map. ATM it is not clear to me + // if that would make the code easier to read. + std::map, vector> d_all; + std::map, vector>> d_sigs; + + // Maybe use a SuffixMatchTree? + std::set d_delegations; + + shared_ptr& d_log; + DNSName d_zone; + time_t d_now; + + bool isRRSetAuth(const DNSName& qname, QType qtype) const; + void parseDRForCache(DNSRecord& dr); + void getByAXFR(const RecZoneToCache::Config&); + void ZoneToCache(const RecZoneToCache::Config& config, uint64_t gen); +}; + +bool ZoneData::isRRSetAuth(const DNSName& qname, QType qtype) const +{ + DNSName delegatedZone(qname); + if (qtype == QType::DS) { + delegatedZone.chopOff(); + } + bool isDelegated = false; + for (;;) { + if (d_delegations.count(delegatedZone) > 0) { + isDelegated = true; + break; + } + delegatedZone.chopOff(); + if (delegatedZone == g_rootdnsname || delegatedZone == d_zone) + break; + } + return !isDelegated; +} + +void ZoneData::parseDRForCache(DNSRecord& dr) +{ + const auto key = pair(dr.d_name, dr.d_type); + + dr.d_ttl += d_now; + + switch (dr.d_type) { + case QType::NSEC: + case QType::NSEC3: + break; + case QType::RRSIG: { + const auto& rr = getRR(dr); + const auto sigkey = pair(key.first, rr->d_type); + auto found = d_sigs.find(sigkey); + if (found != d_sigs.end()) { + found->second.push_back(rr); + } + else { + vector> sigsrr; + sigsrr.push_back(rr); + d_sigs.insert({sigkey, sigsrr}); + } + break; + } + case QType::NS: + if (dr.d_name != d_zone) { + d_delegations.insert(dr.d_name); + } + break; + default: + break; + } + + auto found = d_all.find(key); + if (found != d_all.end()) { + found->second.push_back(dr); + } + else { + vector v; + v.push_back(dr); + d_all.insert({key, v}); + } +} + +void ZoneData::getByAXFR(const RecZoneToCache::Config& config) +{ + ComboAddress primary = ComboAddress(config.d_sources.at(0), 53); + uint16_t axfrTimeout = config.d_timeout; + size_t maxReceivedBytes = config.d_maxReceivedBytes; + const TSIGTriplet tt = config.d_tt; + ComboAddress local = config.d_local; + if (local == ComboAddress()) { + local = pdns::getQueryLocalAddress(primary.sin4.sin_family, 0); + } + + AXFRRetriever axfr(primary, d_zone, tt, &local, maxReceivedBytes, axfrTimeout); + Resolver::res_t nop; + vector chunk; + time_t axfrStart = time(nullptr); + time_t axfrNow = time(nullptr); + + while (axfr.getChunk(nop, &chunk, (axfrStart + axfrTimeout - axfrNow))) { + for (auto& dr : chunk) { + parseDRForCache(dr); + } + axfrNow = time(nullptr); + if (axfrNow < axfrStart || axfrNow - axfrStart > axfrTimeout) { + throw std::runtime_error("Total AXFR time for zoneToCache exceeded!"); + } + } +} + +static std::vector getLinesFromFile(const std::string& file) +{ + + std::vector lines; + std::ifstream stream(file); + if (!stream) { + throw std::runtime_error("Cannot read file: " + file); + } + std::string line; + while (std::getline(stream, line)) { + lines.push_back(line); + } + return lines; +} + +static std::vector getURL(const RecZoneToCache::Config& config) +{ + std::vector lines; +#ifdef HAVE_LIBCURL + MiniCurl mc; + ComboAddress local = config.d_local; + std::string reply = mc.getURL(config.d_sources.at(0), nullptr, local == ComboAddress() ? nullptr : &local, config.d_timeout, false, true); + if (config.d_maxReceivedBytes > 0 && reply.size() > config.d_maxReceivedBytes) { + // We should actually detect this *during* the GET + throw std::runtime_error("Retrieved data exceeds maxReceivedBytes"); + } + std::istringstream stream(reply); + string line; + while (std::getline(stream, line)) { + lines.push_back(line); + } +#else + throw std::runtime_error("url method configured but libcurl not compiled in"); +#endif + return lines; +} + +void ZoneData::ZoneToCache(const RecZoneToCache::Config& config, uint64_t configGeneration) +{ + if (config.d_sources.size() > 1) { + d_log->info("Multiple sources not yet supported, using first"); + } + + // We do not do validation, it will happen on-demand if an Indeterminate record is encountered when the caches are queried + // First scan all records collecting info about delegations ans sigs + // A this moment, we ignore NSEC and NSEC3 records. It is not clear to me yet under which conditions + // they could be entered in into the (neg)cache. + + if (config.d_method == "axfr") { + d_log->info("Getting zone by AXFR"); + getByAXFR(config); + } + else { + vector lines; + if (config.d_method == "url") { + d_log->info("Getting zone by URL"); + lines = getURL(config); + } + else if (config.d_method == "file") { + d_log->info("Getting zone from file"); + lines = getLinesFromFile(config.d_sources.at(0)); + } + DNSResourceRecord drr; + ZoneParserTNG zpt(lines, d_zone); + zpt.setMaxGenerateSteps(1); + zpt.setMaxIncludes(0); + + while (zpt.get(drr)) { + DNSRecord dr(drr); + parseDRForCache(dr); + } + } + + // Extra check before we are touching the cache + auto luaconfsLocal = g_luaconfs.getLocal(); + if (luaconfsLocal->generation != configGeneration) { + return; + } + + // Rerun, now inserting the rrsets into the cache with associated sigs + d_now = time(nullptr); + for (const auto& [key, v] : d_all) { + const auto& [qname, qtype] = key; + switch (qtype) { + case QType::NSEC: + case QType::NSEC3: + break; + case QType::RRSIG: + break; + default: { + vector> sigsrr; + auto it = d_sigs.find(key); + if (it != d_sigs.end()) { + sigsrr = it->second; + } + bool auth = isRRSetAuth(qname, qtype); + // Same decision as updateCacheFromRecords() (we do not test for NSEC since we skip those completely) + if (auth || (qtype == QType::NS || qtype == QType::A || qtype == QType::AAAA || qtype == QType::DS)) { + g_recCache->replace(d_now, qname, qtype, v, sigsrr, + std::vector>(), auth, d_zone); + } + break; + } + } + } +} + +// Config must be a copy, so call by value! +void RecZoneToCache::ZoneToCache(RecZoneToCache::Config config, uint64_t configGeneration) +{ + setThreadName("pdns-r/ztc/" + config.d_zone); + auto luaconfsLocal = g_luaconfs.getLocal(); + auto log = g_slog->withName("ztc")->withValues("zone", Logging::Loggable(config.d_zone)); + + while (true) { + if (luaconfsLocal->generation != configGeneration) { + /* the configuration has been reloaded, meaning that a new thread + has been started to handle that zone and we are now obsolete. + */ + log->info("A more recent configuration has been found, stopping the old update thread"); + return; + } + + time_t refresh = config.d_retryOnError; + try { + ZoneData data(log, config.d_zone); + data.ZoneToCache(config, configGeneration); + if (luaconfsLocal->generation != configGeneration) { + log->info("A more recent configuration has been found, stopping the old update thread"); + return; + } + refresh = config.d_refreshPeriod; + log->info("Loaded zone into cache", "refresh", Logging::Loggable(refresh)); + } + catch (const PDNSException& e) { + log->info("Unable to load zone into cache, will retry", "exception", Logging::Loggable(e.reason), "refresh", Logging::Loggable(refresh)); + } + catch (const std::runtime_error& e) { + log->info("Unable to load zone into cache, will retry", "exception", Logging::Loggable(e.what()), "refresh", Logging::Loggable(refresh)); + } + catch (...) { + log->info("Unable to load zone into cache, will retry", "exception", Logging::Loggable("unknown"), "refresh", Logging::Loggable(refresh)); + } + if (refresh == 0) { + return; // single shot + } + sleep(refresh); + } +} diff --git a/pdns/recursordist/rec-zonetocache.hh b/pdns/recursordist/rec-zonetocache.hh new file mode 100644 index 000000000000..8eb6425af221 --- /dev/null +++ b/pdns/recursordist/rec-zonetocache.hh @@ -0,0 +1,45 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "namespaces.hh" +#include "dns.hh" +#include "iputils.hh" + +class RecZoneToCache +{ +public: + struct Config + { + std::string d_zone; // Zone name + std::string d_method; // axfr, http, https, file + vector d_sources; // IPs or URLs + ComboAddress d_local; // local address + TSIGTriplet d_tt; // Authentication data + size_t d_maxReceivedBytes{0}; // Maximum size + time_t d_retryOnError{60}; // Retry on error + time_t d_refreshPeriod{24 * 3600}; // Time between refetch + uint32_t d_timeout{20}; // timeout in seconds + }; + static void ZoneToCache(Config config, uint64_t gen); +}; diff --git a/pdns/recursordist/recursor-tsan.supp b/pdns/recursordist/recursor-tsan.supp new file mode 100644 index 000000000000..ee6d9eaeea68 --- /dev/null +++ b/pdns/recursordist/recursor-tsan.supp @@ -0,0 +1,3 @@ +# We don't care about stats for now +race:doStats +race:g_stats diff --git a/pdns/recursordist/stable-bloom.hh b/pdns/recursordist/stable-bloom.hh index ee5c7b437584..3fc14cce8986 100644 --- a/pdns/recursordist/stable-bloom.hh +++ b/pdns/recursordist/stable-bloom.hh @@ -126,7 +126,7 @@ public: if (bitstr_len > 2 * 64 * 1024 * 1024U) { // twice the current size throw std::runtime_error("SBF: read failed (bitstr_len too big)"); } - unique_ptr bitcstr = make_unique(bitstr_len); + auto bitcstr = std::make_unique(bitstr_len); is.read(bitcstr.get(), bitstr_len); if (is.fail()) { throw std::runtime_error("SBF: read failed (file too short?)"); diff --git a/pdns/recursordist/taskqueue.hh b/pdns/recursordist/taskqueue.hh index ec0c91d08894..5bd53029b99b 100644 --- a/pdns/recursordist/taskqueue.hh +++ b/pdns/recursordist/taskqueue.hh @@ -21,6 +21,7 @@ */ #pragma once +#include #include #include diff --git a/pdns/recursordist/test-aggressive_nsec_cc.cc b/pdns/recursordist/test-aggressive_nsec_cc.cc index e507145a28fa..51693050c66c 100644 --- a/pdns/recursordist/test-aggressive_nsec_cc.cc +++ b/pdns/recursordist/test-aggressive_nsec_cc.cc @@ -774,13 +774,13 @@ BOOST_AUTO_TEST_CASE(test_aggressive_nsec3_nodata_wildcard) addRecordToLW(res, DNSName("powerdns.com."), QType::SOA, "powerdns.com. powerdns.com. 2017032301 10800 3600 604800 3600", DNSResourceRecord::AUTHORITY, 3600); addRRSIG(keys, res->d_records, DNSName("powerdns.com."), 300); /* first the closest encloser */ - addNSEC3UnhashedRecordToLW(DNSName("powerdns.com."), DNSName("powerdns.com."), "whatever", {QType::A, QType::TXT, QType::RRSIG}, 600, res->d_records, 10); + addNSEC3NoDataNarrowRecordToLW(DNSName("powerdns.com."), DNSName("powerdns.com."), {QType::A, QType::TXT, QType::RRSIG}, 600, res->d_records, 10); addRRSIG(keys, res->d_records, DNSName("powerdns.com."), 300); /* then the next closer */ - addNSEC3UnhashedRecordToLW(DNSName("+.powerdns.com."), DNSName("powerdns.com."), "v", {QType::RRSIG}, 600, res->d_records, 10); + addNSEC3NarrowRecordToLW(DNSName("a.powerdns.com."), DNSName("powerdns.com."), {QType::RRSIG}, 600, res->d_records, 10); addRRSIG(keys, res->d_records, DNSName("powerdns.com."), 300); /* a wildcard applies but does not have this type */ - addNSEC3UnhashedRecordToLW(DNSName("*.powerdns.com."), DNSName("powerdns.com."), "whatever", {QType::TXT, QType::RRSIG}, 600, res->d_records, 10); + addNSEC3NoDataNarrowRecordToLW(DNSName("*.powerdns.com."), DNSName("powerdns.com."), {QType::TXT, QType::RRSIG}, 600, res->d_records, 10); addRRSIG(keys, res->d_records, DNSName("powerdns.com"), 300, false, boost::none, DNSName("*.powerdns.com")); return LWResult::Result::Success; } diff --git a/pdns/recursordist/test-credentials_cc.cc b/pdns/recursordist/test-credentials_cc.cc new file mode 120000 index 000000000000..8f713318b2e2 --- /dev/null +++ b/pdns/recursordist/test-credentials_cc.cc @@ -0,0 +1 @@ +../test-credentials_cc.cc \ No newline at end of file diff --git a/pdns/recursordist/test-ednsoptions_cc.cc b/pdns/recursordist/test-ednsoptions_cc.cc index cb07d377d652..5878022cb009 100644 --- a/pdns/recursordist/test-ednsoptions_cc.cc +++ b/pdns/recursordist/test-ednsoptions_cc.cc @@ -22,17 +22,15 @@ static void getRawQueryWithECSAndCookie(const DNSName& name, const Netmask& ecs, DNSPacketWriter pw(query, name, QType::A, QClass::IN, 0); pw.commit(); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = clientCookie; - cookiesOpt.server = serverCookie; - string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt(clientCookie + serverCookie); + string cookiesOptionStr = cookiesOpt.makeOptString(); EDNSSubnetOpts ecsOpts; ecsOpts.source = ecs; string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); DNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); } diff --git a/pdns/recursordist/test-filterpo_cc.cc b/pdns/recursordist/test-filterpo_cc.cc index 5b48fb568757..9b0ff1fe1909 100644 --- a/pdns/recursordist/test-filterpo_cc.cc +++ b/pdns/recursordist/test-filterpo_cc.cc @@ -33,19 +33,19 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) const DNSName blockedWildcardName("*.wildcard-blocked."); const ComboAddress responseIP("192.0.2.254"); BOOST_CHECK_EQUAL(zone->size(), 0U); - zone->addClientTrigger(Netmask(clientIP, 32), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ClientIP)); + zone->addClientTrigger(Netmask(clientIP, 31), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ClientIP)); BOOST_CHECK_EQUAL(zone->size(), 1U); zone->addQNameTrigger(blockedName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::QName)); BOOST_CHECK_EQUAL(zone->size(), 2U); zone->addQNameTrigger(blockedWildcardName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::QName)); BOOST_CHECK_EQUAL(zone->size(), 3U); - zone->addNSIPTrigger(Netmask(nsIP, 32), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSIP)); + zone->addNSIPTrigger(Netmask(nsIP, 31), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSIP)); BOOST_CHECK_EQUAL(zone->size(), 4U); zone->addNSTrigger(nsName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSDName)); BOOST_CHECK_EQUAL(zone->size(), 5U); zone->addNSTrigger(nsWildcardName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSDName)); BOOST_CHECK_EQUAL(zone->size(), 6U); - zone->addResponseTrigger(Netmask(responseIP, 32), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ResponseIP)); + zone->addResponseTrigger(Netmask(responseIP, 31), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ResponseIP)); BOOST_CHECK_EQUAL(zone->size(), 7U); size_t zoneIdx = dfe.addZone(zone); @@ -81,6 +81,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) const auto matchingPolicy = dfe.getProcessingPolicy(DNSName("sub.sub.wildcard.wolf."), std::unordered_map(), DNSFilterEngine::maximumPriority); BOOST_CHECK(matchingPolicy.d_type == DNSFilterEngine::PolicyType::NSDName); BOOST_CHECK(matchingPolicy.d_kind == DNSFilterEngine::PolicyKind::Drop); + BOOST_CHECK_EQUAL(matchingPolicy.d_trigger, DNSName("*.wildcard.wolf.rpz-nsdname")); + BOOST_CHECK_EQUAL(matchingPolicy.d_hit, "sub.sub.wildcard.wolf"); /* looking for wildcard.wolf. should not match *.wildcard-blocked. */ const auto notMatchingPolicy = dfe.getProcessingPolicy(DNSName("wildcard.wolf."), std::unordered_map(), DNSFilterEngine::maximumPriority); @@ -92,6 +94,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) /* except if we look exactly for the wildcard */ BOOST_CHECK(zone->findExactNSPolicy(nsWildcardName, zonePolicy)); BOOST_CHECK(zonePolicy == matchingPolicy); + BOOST_CHECK_EQUAL(zonePolicy.d_trigger, DNSName("*.wildcard.wolf.rpz-nsdname")); + BOOST_CHECK_EQUAL(zonePolicy.d_hit, nsWildcardName.toStringNoDot()); } { @@ -110,6 +114,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) DNSFilterEngine::Policy zonePolicy; BOOST_CHECK(zone->findNSIPPolicy(nsIP, zonePolicy)); BOOST_CHECK(zonePolicy == matchingPolicy); + BOOST_CHECK_EQUAL(zonePolicy.d_trigger, DNSName("31.0.2.0.192.rpz-nsip")); + BOOST_CHECK_EQUAL(zonePolicy.d_hit, nsIP.toString()); } { @@ -128,6 +134,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) DNSFilterEngine::Policy zonePolicy; BOOST_CHECK(zone->findExactQNamePolicy(blockedName, zonePolicy)); BOOST_CHECK(zonePolicy == matchingPolicy); + BOOST_CHECK_EQUAL(zonePolicy.d_trigger, blockedName); + BOOST_CHECK_EQUAL(zonePolicy.d_hit, blockedName.toStringNoDot()); /* but a subdomain should not be blocked (not a wildcard, and this is not suffix domain matching */ matchingPolicy = dfe.getQueryPolicy(DNSName("sub") + blockedName, std::unordered_map(), DNSFilterEngine::maximumPriority); @@ -140,6 +148,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) const auto matchingPolicy = dfe.getQueryPolicy(DNSName("sub.sub.wildcard-blocked."), std::unordered_map(), DNSFilterEngine::maximumPriority); BOOST_CHECK(matchingPolicy.d_type == DNSFilterEngine::PolicyType::QName); BOOST_CHECK(matchingPolicy.d_kind == DNSFilterEngine::PolicyKind::Drop); + BOOST_CHECK_EQUAL(matchingPolicy.d_trigger, blockedWildcardName); + BOOST_CHECK_EQUAL(matchingPolicy.d_hit, "sub.sub.wildcard-blocked"); /* looking for wildcard-blocked. should not match *.wildcard-blocked. */ const auto notMatchingPolicy = dfe.getQueryPolicy(DNSName("wildcard-blocked."), std::unordered_map(), DNSFilterEngine::maximumPriority); @@ -151,6 +161,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) /* except if we look exactly for the wildcard */ BOOST_CHECK(zone->findExactQNamePolicy(blockedWildcardName, zonePolicy)); BOOST_CHECK(zonePolicy == matchingPolicy); + BOOST_CHECK_EQUAL(zonePolicy.d_trigger, blockedWildcardName); + BOOST_CHECK_EQUAL(zonePolicy.d_hit, blockedWildcardName.toStringNoDot()); } { @@ -161,6 +173,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) DNSFilterEngine::Policy zonePolicy; BOOST_CHECK(zone->findClientPolicy(clientIP, zonePolicy)); BOOST_CHECK(zonePolicy == matchingPolicy); + BOOST_CHECK_EQUAL(zonePolicy.d_trigger, DNSName("31.128.2.0.192.rpz-client-ip")); + BOOST_CHECK_EQUAL(zonePolicy.d_hit, clientIP.toString()); } { @@ -183,6 +197,8 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) DNSFilterEngine::Policy zonePolicy; BOOST_CHECK(zone->findResponsePolicy(responseIP, zonePolicy)); BOOST_CHECK(zonePolicy == matchingPolicy); + BOOST_CHECK_EQUAL(zonePolicy.d_trigger, DNSName("31.254.2.0.192.rpz-ip")); + BOOST_CHECK_EQUAL(zonePolicy.d_hit, responseIP.toString()); } { @@ -197,19 +213,19 @@ BOOST_AUTO_TEST_CASE(test_filter_policies_basic) } BOOST_CHECK_EQUAL(zone->size(), 7U); - zone->rmClientTrigger(Netmask(clientIP, 32), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ClientIP)); + zone->rmClientTrigger(Netmask(clientIP, 31), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ClientIP)); BOOST_CHECK_EQUAL(zone->size(), 6U); zone->rmQNameTrigger(blockedName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::QName)); BOOST_CHECK_EQUAL(zone->size(), 5U); zone->rmQNameTrigger(blockedWildcardName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::QName)); BOOST_CHECK_EQUAL(zone->size(), 4U); - zone->rmNSIPTrigger(Netmask(nsIP, 32), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSIP)); + zone->rmNSIPTrigger(Netmask(nsIP, 31), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSIP)); BOOST_CHECK_EQUAL(zone->size(), 3U); zone->rmNSTrigger(nsName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSDName)); BOOST_CHECK_EQUAL(zone->size(), 2U); zone->rmNSTrigger(nsWildcardName, DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::NSDName)); BOOST_CHECK_EQUAL(zone->size(), 1U); - zone->rmResponseTrigger(Netmask(responseIP, 32), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ResponseIP)); + zone->rmResponseTrigger(Netmask(responseIP, 31), DNSFilterEngine::Policy(DNSFilterEngine::PolicyKind::Drop, DNSFilterEngine::PolicyType::ResponseIP)); BOOST_CHECK_EQUAL(zone->size(), 0U); /* DNSFilterEngine::clear() calls clear() on all zones, but keeps the zones */ diff --git a/pdns/recursordist/test-histogram_hh.cc b/pdns/recursordist/test-histogram_hh.cc index 3858a6ec74b6..b336df55d55d 100644 --- a/pdns/recursordist/test-histogram_hh.cc +++ b/pdns/recursordist/test-histogram_hh.cc @@ -43,7 +43,7 @@ BOOST_AUTO_TEST_CASE(test_simple) h(4); h(100); h(101); - h(-1); + h(-1); // actually a very large value, but for sum it will boil down to be -1 */ auto data = h.getRawData(); BOOST_CHECK_EQUAL(data.size(), 6U); @@ -60,6 +60,7 @@ BOOST_AUTO_TEST_CASE(test_simple) for (auto e : cexpected) { BOOST_CHECK_EQUAL(c[i++], e); } + BOOST_CHECK_EQUAL(h.getSum(), 209U); } BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/recursordist/test-rec-zonetocache.cc b/pdns/recursordist/test-rec-zonetocache.cc new file mode 100644 index 000000000000..3dde7d1237a6 --- /dev/null +++ b/pdns/recursordist/test-rec-zonetocache.cc @@ -0,0 +1,92 @@ +#define BOOST_TEST_DYN_LINK +#include + +#include + +#include "rec-zonetocache.hh" +#include "recursor_cache.hh" + +extern unique_ptr g_recCache; + +BOOST_AUTO_TEST_SUITE(rec_zonetocache) + +// A piece of the root zone +const std::string zone = ". 86400 IN SOA a.root-servers.net. nstld.verisign-grs.com. 2021080900 1800 900 604800 86400\n" + ". 86400 IN RRSIG SOA 8 0 86400 20210822050000 20210809040000 26838 . IYvQp/mwMY9i8zGwc1azfS1lSHunX5SstuVXfZ5bNaZTp45SGLF7uzIAzX+/o1Dl5rKH27R8CE26dzTi5ds/OqsGrYXsTcdhwOQEmhYnfmKjGPFqfl0A6MLhHuyEkydPDdVxr5oMIEsIm0hMR1QDZS5gfpmcCfwTbkRGJP53yVnBAMHKhCK40k49YzN0PIycTBZgXu7dGWL2cfYlND1WLImcs1GPqu/8ND9tiLbhoDRw85kqLLR7IGPVb5xu/pNOfpX2nNkvBzC4M7bOaUAquxxetBSjMiPILSkC0XXsnQW7rsei7cmktt2CXU2IYqWkho8pe4A849TCzo1+Aheglw==\n" + ". 518400 IN NS a.root-servers.net.\n" + ". 518400 IN NS b.root-servers.net.\n" + ". 518400 IN NS c.root-servers.net.\n" + ". 518400 IN NS d.root-servers.net.\n" + ". 518400 IN NS e.root-servers.net.\n" + ". 518400 IN NS f.root-servers.net.\n" + ". 518400 IN NS g.root-servers.net.\n" + ". 518400 IN NS h.root-servers.net.\n" + ". 518400 IN NS i.root-servers.net.\n" + ". 518400 IN NS j.root-servers.net.\n" + ". 518400 IN NS k.root-servers.net.\n" + ". 518400 IN NS l.root-servers.net.\n" + ". 518400 IN NS m.root-servers.net.\n" + ". 518400 IN RRSIG NS 8 0 518400 20210822050000 20210809040000 26838 . rXMbGiqM2MOKVpykT064JBDF+4lrSisIWsL5Ro5pOnKJ4AELxsUSjFXRHtCpd6Ii3FUkdDUlhi4cGhWVK918sGadLKlVyks9IET1evutYk5u5w4EuUbaFelpLnGeT78QF6rbXRji3TDs3QeHQ8VqpHTnTiFSDVqIllTqF0DseiuIZ1IgT/Ho8PeX2oXAPEHfqcZbm3vmfZL8Ju+RyvbFpJ6f9AkuWMikIWjyz7xooNwJBjHtj3omIUo3BP+acbigSwmxwZaDgCbzyvtLxAU62WyNWeaOe6O5zwLR3fjhwCMBuVBm1qrA9Nmc9s5l/QzgIoI2SfE1/G7iwdwfXuU//g==\n" + ". 86400 IN NSEC aaa. NS SOA RRSIG NSEC DNSKEY\n" + ". 86400 IN RRSIG NSEC 8 0 86400 20210822050000 20210809040000 26838 . QQHXJDAkTPyrfP4MRSRsNz5AbVyYzPkyg3vCbUq9w144Bd0EjaXxGxd6zZ0dXVCKmf9UUpUvdbposIVqHeA3LLKmgRHXgkFMYxu2LbxoR+dnrcvNVBM/QFb+cHzQWLGl+D2is21UoUcfDKnPMNiWTxkUJNW4mTdVVIkck8FiBSwslrS4eD8irQJv9s6TbS9VYZQcI4UOqkd3XaFRox1UjzHkAVqN8Rv4O4IV0EW8yKGoMOJew/JBl8KCmWgqJDee1oMf6h2ZhBrwmF+uRiatl3wcfEGnquWKdq2ZSdzas6RbbO4T/oiJYoEL1TGmQ5hxzcbs+6AE+ixRk9m3+B7uuw==\n" + ". 172800 IN DNSKEY 256 3 8 AwEAAbDEyqdwu2fqAwinPCFwALUCWfYYaLrNhnOrMxDorLBYMipEE1btlK1XnigTRMeb0YQ8/LCopb3CN73hYDhCHFsNk+GtukBB+gWLcg+2FZXbhLXIheQm8x2VfOHy2yYQG+18wjx3HY9Mj/ZEhXbZNrDMvpFKKVihWXa0/cHNg4ZcIHD9KkMlKzK+my1K/vz8fq5cFCFOu7wgM+kKbOikdcRBm7Uf/wRXZItFg2uhUijUb56gEN8uCUgmuEw6wQ5ZBuR7UT/FLyyAUeAH87oxF4im2DXK6J+JA7IAs2UHJ16uTqvdserUU8NIosislaXIZCvz+NTDb3SJcxs6bvCikeU=\n" + ". 172800 IN DNSKEY 257 3 8 AwEAAaz/tAm8yTn4Mfeh5eyI96WSVexTBAvkMgJzkKTOiW1vkIbzxeF3+/4RgWOq7HrxRixHlFlExOLAJr5emLvN7SWXgnLh4+B5xQlNVz8Og8kvArMtNROxVQuCaSnIDdD5LKyWbRd2n9WGe2R8PzgCmr3EgVLrjyBxWezF0jLHwVN8efS3rCj/EWgvIWgb9tarpVUDK/b58Da+sqqls3eNbuv7pr+eoZG+SrDK6nWeL3c6H5Apxz7LjVc1uTIdsIXxuOLYA4/ilBmSVIzuDWfdRUfhHdY6+cn8HFRm+2hM8AnXGXws9555KrUB5qihylGa8subX2Nn6UwNR1AkUTV74bU=\n" + ". 172800 IN RRSIG DNSKEY 8 0 172800 20210821000000 20210731000000 20326 . MQCiL1+LRbTdEm3hyrUQOjMurN4QNmB7Up8ahSMS3vC+Waa7ywqKnClMNiZqPJKYuAn3wHZM0kj5pTF/wASaLJ9duxmcL6Wyi7mugRZE1Pv3RDvp6o4RbroZND9HVnImDxg/GmaJbRxtPuA/dGSp/Iq7AObJrtgNM81p19Z50Zn89POIr50/7SpuiRZ7xFYhfwIug9pgrHHdNQpQN7UVUJVXMDgtH/IT8ACgvGfoVLXCciPIpKJhwYZXq9nOGBgVxV7/h7aIGkrOryfm2ChvIei3MQANyjiq1QfckaXHpg+iHa4J6kduZoTR5Pe+E0VFpf8q63kYBaJTshOVVMm3TQ==\n" + "aaa. 172800 IN NS ns1.dns.nic.aaa.\n" + "aaa. 172800 IN NS ns2.dns.nic.aaa.\n" + "aaa. 172800 IN NS ns3.dns.nic.aaa.\n" + "aaa. 172800 IN NS ns4.dns.nic.aaa.\n" + "aaa. 172800 IN NS ns5.dns.nic.aaa.\n" + "aaa. 172800 IN NS ns6.dns.nic.aaa.\n" + "aaa. 86400 IN DS 1657 8 1 0B0D56361CE62118537E07A680E9582F5F5FA129\n" + "aaa. 86400 IN DS 1657 8 2 9D6BAE62219231C99FAA479716B6E4619330CE8206670AEA6C1673A055DC3AF2\n" + "aaa. 86400 IN RRSIG DS 8 1 86400 20210822050000 20210809040000 26838 . mZKBrX0gtfNXQ0VhUDcfmat7isb4YqGe110YF6VEdGcQZbcEjb+fsPBSriiHcBMcncOc57f+H0HDERe6Y0XlW8ZvLOPfH1AzOMHcc21Qgt0Zow4Dt06YAzX0ONw4FJUPyoHy2PPJhgldv6vywzHe9FzrAEoc/XB64tZjvNIp87HVP+YRhLi+3EFkWlnwhDJa/xvVGpPppyO+AX5Nh3VdWJ0awt72BwgZOyiqDgtfEjv3w1RGBa86I4hQ8QYR6PM9ghAPLhE/nMYnktzufTgnLpNhqhvWEvBDZlzXnRr51ZhRq2RZ/z6cdgnPITZM7aJCJfMZagRhLXujCWVrH9MFbw==\n" + "aaa. 86400 IN NSEC aarp. NS DS RRSIG NSEC\n" + "aaa. 86400 IN RRSIG NSEC 8 1 86400 20210822050000 20210809040000 26838 . iII7liwvc+VAAQRPVzeSl1kDRbYe3euoIyoVKdeHctNiXBef3TUkEQPytwkOSPmsrsIImQXiUo5RQWEjZdFCa4o1UJTeiDIrzYCu4sDdgtiSHkgsTdS6z9OmWCuyj1fLRaLBblRfZsGa+ObIU/IwhC+jm38SuCTPR3wwaizo934ck082zaUdmhioEXHq/wAum/Za9rNp8gJpL1GUib7U8BupwchWDcn6+VG5CnKN0N/R52K2PfEJ8JdXC8mM2Q0xJYN3aptPLqJzIqfstK874Hcsdw3Z5cG2LXTQZsnvqCg74tCwS6JgeYRPu/P7+yMoH349Ib/7WFGmM3iYW7M87w==\n" + "ns1.dns.nic.aaa. 172800 IN A 156.154.144.2\n" + "ns1.dns.nic.aaa. 172800 IN AAAA 2610:a1:1071:0:0:0:0:2\n" + "ns2.dns.nic.aaa. 172800 IN A 156.154.145.2\n" + "ns2.dns.nic.aaa. 172800 IN AAAA 2610:a1:1072:0:0:0:0:2\n" + "ns3.dns.nic.aaa. 172800 IN A 156.154.159.2\n" + "ns3.dns.nic.aaa. 172800 IN AAAA 2610:a1:1073:0:0:0:0:2\n" + "ns4.dns.nic.aaa. 172800 IN A 156.154.156.2\n" + "ns4.dns.nic.aaa. 172800 IN AAAA 2610:a1:1074:0:0:0:0:2\n" + "ns5.dns.nic.aaa. 172800 IN A 156.154.157.2\n" + "ns5.dns.nic.aaa. 172800 IN AAAA 2610:a1:1075:0:0:0:0:2\n" + "ns6.dns.nic.aaa. 172800 IN A 156.154.158.2\n" + "ns6.dns.nic.aaa. 172800 IN AAAA 2610:a1:1076:0:0:0:0:2\n"; + +BOOST_AUTO_TEST_CASE(test_zonetocache) +{ + //g_slog = Logging::Logger::create(loggerBackend); + + char temp[] = "/tmp/ztcXXXXXXXXXX"; + int fd = mkstemp(temp); + BOOST_REQUIRE(fd > 0); + FILE* fp = fdopen(fd, "w"); + BOOST_REQUIRE(fp != nullptr); + size_t written = fwrite(zone.data(), 1, zone.length(), fp); + BOOST_REQUIRE(written == zone.length()); + BOOST_REQUIRE(fclose(fp) == 0); + + RecZoneToCache::Config config{".", "file", {temp}, ComboAddress(), TSIGTriplet()}; + config.d_refreshPeriod = 0; + + // Start with a new, empty cache + g_recCache = std::make_unique(); + BOOST_CHECK_EQUAL(g_recCache->size(), 0U); + RecZoneToCache::ZoneToCache(config, 0); + unlink(temp); + BOOST_CHECK_EQUAL(g_recCache->size(), 17U); + + std::vector retrieved; + time_t now = time(nullptr); + ComboAddress who; + BOOST_CHECK_GT(g_recCache->get(now, DNSName("."), QType::SOA, true, &retrieved, who), 0); + // not auth + BOOST_CHECK_LT(g_recCache->get(now, DNSName("aaa."), QType::NS, true, &retrieved, who), 0); + // auth + BOOST_CHECK_GT(g_recCache->get(now, DNSName("aaa."), QType::NS, false, &retrieved, who), 0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/recursordist/test-syncres_cc.cc b/pdns/recursordist/test-syncres_cc.cc index 6a461506393e..7b5d47c61afd 100644 --- a/pdns/recursordist/test-syncres_cc.cc +++ b/pdns/recursordist/test-syncres_cc.cc @@ -14,8 +14,8 @@ GlobalStateHolder g_xdnssec; GlobalStateHolder g_dontThrottleNames; GlobalStateHolder g_dontThrottleNetmasks; GlobalStateHolder g_DoTToAuthNames; -std::unique_ptr g_recCache{nullptr}; -std::unique_ptr g_negCache{nullptr}; +std::unique_ptr g_recCache; +std::unique_ptr g_negCache; unsigned int g_numThreads = 1; bool g_lowercaseOutgoing = false; @@ -27,7 +27,7 @@ ArgvMap& arg() return theArg; } -void primeRootNSZones(bool, unsigned int) +void primeRootNSZones(DNSSECMode, unsigned int) { } @@ -39,7 +39,7 @@ void BaseLua4::getFeatures(Features&) { } -bool RecursorLua4::preoutquery(const ComboAddress& ns, const ComboAddress& requestor, const DNSName& query, const QType& qtype, bool isTcp, vector& res, int& ret) const +bool RecursorLua4::preoutquery(const ComboAddress& ns, const ComboAddress& requestor, const DNSName& query, const QType& qtype, bool isTcp, vector& res, int& ret, RecEventTrace& et) const { return false; } @@ -82,9 +82,9 @@ bool primeHints(time_t now) { vector nsset; if (!g_recCache) - g_recCache = std::unique_ptr(new MemRecursorCache()); + g_recCache = std::make_unique(); if (!g_negCache) - g_negCache = std::unique_ptr(new NegCache()); + g_negCache = std::make_unique(); DNSRecord arr, aaaarr, nsrr; nsrr.d_name = g_rootdnsname; @@ -142,8 +142,8 @@ void initSR(bool debug) g_log.toConsole(Logger::Error); } - g_recCache = std::unique_ptr(new MemRecursorCache()); - g_negCache = std::unique_ptr(new NegCache()); + g_recCache = std::make_unique(); + g_negCache = std::make_unique(); SyncRes::s_maxqperq = 50; SyncRes::s_maxnsaddressqperq = 10; @@ -227,7 +227,7 @@ void initSR(std::unique_ptr& sr, bool dnssec, bool debug, time_t fakeNo initSR(debug); - sr = std::unique_ptr(new SyncRes(now)); + sr = std::make_unique(now); sr->setDoEDNS0(true); if (dnssec) { sr->setDoDNSSEC(dnssec); @@ -431,6 +431,17 @@ void addNSEC3UnhashedRecordToLW(const DNSName& domain, const DNSName& zone, cons addNSEC3RecordToLW(DNSName(toBase32Hex(hashed)) + zone, next, salt, iterations, types, ttl, records, optOut); } +/* Proves a NODATA (name exists, type does not) but the next owner name is right behind, so it should not prove anything else unless we are very unlucky */ +void addNSEC3NoDataNarrowRecordToLW(const DNSName& domain, const DNSName& zone, const std::set& types, uint32_t ttl, std::vector& records, unsigned int iterations, bool optOut) +{ + static const std::string salt = "deadbeef"; + std::string hashed = hashQNameWithSalt(salt, iterations, domain); + std::string hashedNext(hashed); + incrementHash(hashedNext); + + addNSEC3RecordToLW(DNSName(toBase32Hex(hashed)) + zone, hashedNext, salt, iterations, types, ttl, records, optOut); +} + void addNSEC3NarrowRecordToLW(const DNSName& domain, const DNSName& zone, const std::set& types, uint32_t ttl, std::vector& records, unsigned int iterations, bool optOut) { static const std::string salt = "deadbeef"; diff --git a/pdns/recursordist/test-syncres_cc.hh b/pdns/recursordist/test-syncres_cc.hh index dbd05f7c9863..f33abd77cfe4 100644 --- a/pdns/recursordist/test-syncres_cc.hh +++ b/pdns/recursordist/test-syncres_cc.hh @@ -59,8 +59,13 @@ void addNSECRecordToLW(const DNSName& domain, const DNSName& next, const std::se void addNSEC3RecordToLW(const DNSName& hashedName, const std::string& hashedNext, const std::string& salt, unsigned int iterations, const std::set& types, uint32_t ttl, std::vector& records, bool optOut = false); +/* Proves a NODATA (name exists, type does not) */ void addNSEC3UnhashedRecordToLW(const DNSName& domain, const DNSName& zone, const std::string& next, const std::set& types, uint32_t ttl, std::vector& records, unsigned int iterations = 10, bool optOut = false); +/* Proves a NODATA (name exists, type does not) and the next owner name is right behind, so it should not prove anything else unless we are very unlucky */ +void addNSEC3NoDataNarrowRecordToLW(const DNSName& domain, const DNSName& zone, const std::set& types, uint32_t ttl, std::vector& records, unsigned int iterations = 10, bool optOut = false); + +/* Proves a NXDOMAIN (name does not exist) with the owner name right before, and the next name right after, so it should not prove anything else unless we are very unlucky */ void addNSEC3NarrowRecordToLW(const DNSName& domain, const DNSName& zone, const std::set& types, uint32_t ttl, std::vector& records, unsigned int iterations = 10, bool OptOut = false); void generateKeyMaterial(const DNSName& name, unsigned int algo, uint8_t digest, testkeysset_t& keys); diff --git a/pdns/recursordist/test-syncres_cc1.cc b/pdns/recursordist/test-syncres_cc1.cc index f952345e4746..d31c24babe3b 100644 --- a/pdns/recursordist/test-syncres_cc1.cc +++ b/pdns/recursordist/test-syncres_cc1.cc @@ -1936,6 +1936,110 @@ BOOST_AUTO_TEST_CASE(test_dname_dnssec_secure) BOOST_CHECK_EQUAL(ret[4].d_name, cnameTarget); } +BOOST_AUTO_TEST_CASE(test_dname_plus_ns_dnssec_secure) +{ + std::unique_ptr sr; + initSR(sr, true); + setDNSSECValidation(sr, DNSSECMode::ValidateAll); + + primeHints(); + + const DNSName dnameOwner("powerdns"); + const DNSName dnameTarget("example"); + + const DNSName target("dname.powerdns"); + const DNSName cnameTarget("dname.example"); + + testkeysset_t keys; + + auto luaconfsCopy = g_luaconfs.getCopy(); + luaconfsCopy.dsAnchors.clear(); + generateKeyMaterial(g_rootdnsname, DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys, luaconfsCopy.dsAnchors); + generateKeyMaterial(dnameTarget, DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys); + g_luaconfs.setState(luaconfsCopy); + + size_t queries = 0; + + sr->setAsyncCallback([dnameOwner, dnameTarget, target, cnameTarget, keys, &queries](const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, LWResult* res, bool* chained) { + queries++; + + if (type == QType::DS || type == QType::DNSKEY) { + return genericDSAndDNSKEYHandler(res, domain, domain, type, keys, false); + } + + if (domain.isPartOf(dnameOwner)) { + setLWResult(res, 0, true, false, true); + addRecordToLW(res, dnameOwner, QType::DNAME, dnameTarget.toString()); + addRRSIG(keys, res->d_records, DNSName("."), 300); + addRecordToLW(res, domain, QType::CNAME, cnameTarget.toString()); // CNAME from a DNAME is not signed + + addRecordToLW(res, dnameTarget, QType::NS, "a.gtld-servers.net.", DNSResourceRecord::AUTHORITY, 172800); + addDS(dnameTarget, 300, res->d_records, keys); + addRRSIG(keys, res->d_records, DNSName("."), 300); + addRecordToLW(res, "a.gtld-servers.net.", QType::A, "192.0.2.1", DNSResourceRecord::ADDITIONAL, 3600); + return LWResult::Result::Success; + } + else if (domain == cnameTarget) { + setLWResult(res, 0, true, false, true); + addRecordToLW(res, domain, QType::A, "192.0.2.42"); + addRRSIG(keys, res->d_records, dnameTarget, 300); + return LWResult::Result::Success; + } + return LWResult::Result::Timeout; + }); + + vector ret; + int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); + + BOOST_CHECK_EQUAL(res, RCode::NoError); + BOOST_CHECK_EQUAL(sr->getValidationState(), vState::Secure); + BOOST_REQUIRE_EQUAL(ret.size(), 5U); /* DNAME + RRSIG(DNAME) + CNAME + A + RRSIG(A) */ + + BOOST_CHECK_EQUAL(queries, 4U); + + BOOST_REQUIRE(ret[0].d_type == QType::DNAME); + BOOST_CHECK(ret[0].d_name == dnameOwner); + BOOST_CHECK_EQUAL(getRR(ret[0])->getTarget(), dnameTarget); + + BOOST_REQUIRE(ret[1].d_type == QType::RRSIG); + BOOST_CHECK_EQUAL(ret[1].d_name, dnameOwner); + + BOOST_CHECK(ret[2].d_type == QType::CNAME); + BOOST_CHECK_EQUAL(ret[2].d_name, target); + + BOOST_CHECK(ret[3].d_type == QType::A); + BOOST_CHECK_EQUAL(ret[3].d_name, cnameTarget); + + BOOST_CHECK(ret[4].d_type == QType::RRSIG); + BOOST_CHECK_EQUAL(ret[4].d_name, cnameTarget); + + // And the cache + ret.clear(); + res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); + + BOOST_CHECK_EQUAL(res, RCode::NoError); + BOOST_CHECK_EQUAL(sr->getValidationState(), vState::Secure); + BOOST_REQUIRE_EQUAL(ret.size(), 5U); /* DNAME + RRSIG(DNAME) + CNAME + A + RRSIG(A) */ + + BOOST_CHECK_EQUAL(queries, 4U); + + BOOST_REQUIRE(ret[0].d_type == QType::DNAME); + BOOST_CHECK(ret[0].d_name == dnameOwner); + BOOST_CHECK_EQUAL(getRR(ret[0])->getTarget(), dnameTarget); + + BOOST_CHECK(ret[1].d_type == QType::RRSIG); + BOOST_CHECK_EQUAL(ret[1].d_name, dnameOwner); + + BOOST_CHECK(ret[2].d_type == QType::CNAME); + BOOST_CHECK_EQUAL(ret[2].d_name, target); + + BOOST_CHECK(ret[3].d_type == QType::A); + BOOST_CHECK_EQUAL(ret[3].d_name, cnameTarget); + + BOOST_CHECK(ret[4].d_type == QType::RRSIG); + BOOST_CHECK_EQUAL(ret[4].d_name, cnameTarget); +} + BOOST_AUTO_TEST_CASE(test_dname_dnssec_insecure) { /* diff --git a/pdns/recursordist/test-syncres_cc2.cc b/pdns/recursordist/test-syncres_cc2.cc index 03d52c107189..8e46deb05fd6 100644 --- a/pdns/recursordist/test-syncres_cc2.cc +++ b/pdns/recursordist/test-syncres_cc2.cc @@ -1107,7 +1107,7 @@ BOOST_AUTO_TEST_CASE(test_rfc8020_nodata_bis) BOOST_CHECK_EQUAL(g_negCache->size(), 2U); } -BOOST_AUTO_TEST_CASE(test_skip_negcache_for_variable_response) +BOOST_AUTO_TEST_CASE(test_dont_skip_negcache_for_variable_response) { std::unique_ptr sr; initSR(sr); @@ -1160,8 +1160,7 @@ BOOST_AUTO_TEST_CASE(test_skip_negcache_for_variable_response) int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); BOOST_CHECK_EQUAL(res, RCode::NXDomain); BOOST_CHECK_EQUAL(ret.size(), 2U); - /* no negative cache entry because the response was variable */ - BOOST_CHECK_EQUAL(g_negCache->size(), 0U); + BOOST_CHECK_EQUAL(g_negCache->size(), 1U); } BOOST_AUTO_TEST_CASE(test_ecs_cache_limit_allowed) diff --git a/pdns/recursordist/test-syncres_cc3.cc b/pdns/recursordist/test-syncres_cc3.cc index 57679ab67a7e..bcd6ddbb2324 100644 --- a/pdns/recursordist/test-syncres_cc3.cc +++ b/pdns/recursordist/test-syncres_cc3.cc @@ -109,6 +109,133 @@ BOOST_AUTO_TEST_CASE(test_no_data_qmin) test_no_data_f(true); } +BOOST_AUTO_TEST_CASE(test_extra_answers) +{ + // Test extra records in the answer section + std::unique_ptr sr; + initSR(sr); + + primeHints(); + + const DNSName target("www.powerdns.com."); + const DNSName target2("www2.powerdns.com."); // in bailiwick, but not asked for + const DNSName target3("www.random.net."); // out of bailiwick and not asked for + + sr->setAsyncCallback([target, target2, target3](const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, LWResult* res, bool* chained) { + if (isRootServer(ip)) { + setLWResult(res, 0, false, false, true); + addRecordToLW(res, "powerdns.com.", QType::NS, "a.gtld-servers.net.", DNSResourceRecord::AUTHORITY, 172800); + addRecordToLW(res, "a.gtld-servers.net.", QType::A, "192.0.2.1", DNSResourceRecord::ADDITIONAL, 3600); + return LWResult::Result::Success; + } + + setLWResult(res, 0, true, false, true); + addRecordToLW(res, domain, QType::A, "192.0.2.2", DNSResourceRecord::ANSWER, 10); + addRecordToLW(res, target2, QType::A, "192.0.2.3", DNSResourceRecord::ANSWER, 10); + addRecordToLW(res, target3, QType::A, "192.0.2.4", DNSResourceRecord::ANSWER, 10); + + return LWResult::Result::Success; + }); + + const time_t now = sr->getNow().tv_sec; + + // we should only see a single record for the question we asked + vector ret; + int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); + BOOST_CHECK_EQUAL(res, RCode::NoError); + BOOST_REQUIRE_EQUAL(ret.size(), 1U); + BOOST_REQUIRE_EQUAL(QType(ret.at(0).d_type).toString(), QType(QType::A).toString()); + BOOST_CHECK_EQUAL(getRR(ret.at(0))->getCA().toString(), ComboAddress("192.0.2.2").toString()); + + // Also check the cache + const ComboAddress who; + vector cached; + BOOST_REQUIRE_GT(g_recCache->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_EQUAL(cached.size(), 1U); + BOOST_REQUIRE_EQUAL(QType(cached.at(0).d_type).toString(), QType(QType::A).toString()); + BOOST_CHECK_EQUAL(getRR(cached.at(0))->getCA().toString(), ComboAddress("192.0.2.2").toString()); + + // The cache should also have an authoritative record for the extra in-bailiwick record + BOOST_REQUIRE_GT(g_recCache->get(now, target2, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_EQUAL(cached.size(), 1U); + BOOST_REQUIRE_EQUAL(QType(cached.at(0).d_type).toString(), QType(QType::A).toString()); + BOOST_CHECK_EQUAL(getRR(cached.at(0))->getCA().toString(), ComboAddress("192.0.2.3").toString()); + + // But the out-of-bailiwick record should not be there + BOOST_REQUIRE_LT(g_recCache->get(now, target3, QType(QType::A), true, &cached, who), 0); +} + +BOOST_AUTO_TEST_CASE(test_dnssec_extra_answers) +{ + // Test extra records in the answer section + std::unique_ptr sr; + initSR(sr, true); + + setDNSSECValidation(sr, DNSSECMode::ValidateAll); + + primeHints(); + const DNSName target("www.powerdns.com."); + const DNSName target2("www2.powerdns.com."); // in bailiwick, but not asked for + const DNSName target3("www.random.net."); // out of bailiwick and not asked for + testkeysset_t keys; + + auto luaconfsCopy = g_luaconfs.getCopy(); + luaconfsCopy.dsAnchors.clear(); + generateKeyMaterial(g_rootdnsname, DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys, luaconfsCopy.dsAnchors); + generateKeyMaterial(DNSName("powerdns.com"), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys, luaconfsCopy.dsAnchors); + g_luaconfs.setState(luaconfsCopy); + + sr->setAsyncCallback([target, target2, target3, keys](const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, LWResult* res, bool* chained) { + if (type == QType::DS || type == QType::DNSKEY) { + return genericDSAndDNSKEYHandler(res, domain, domain, type, keys, false); + } + if (isRootServer(ip)) { + setLWResult(res, 0, false, false, true); + addRecordToLW(res, "powerdns.com.", QType::NS, "a.gtld-servers.net.", DNSResourceRecord::AUTHORITY, 172800); + addRRSIG(keys, res->d_records, DNSName("."), 300); + addRecordToLW(res, "a.gtld-servers.net.", QType::A, "192.0.2.1", DNSResourceRecord::ADDITIONAL, 3600); + return LWResult::Result::Success; + } + + setLWResult(res, 0, true, false, true); + addRecordToLW(res, domain, QType::A, "192.0.2.2", DNSResourceRecord::ANSWER, 10); + addRRSIG(keys, res->d_records, DNSName("powerdns.com"), 300); + addRecordToLW(res, target2, QType::A, "192.0.2.3", DNSResourceRecord::ANSWER, 10); + addRRSIG(keys, res->d_records, DNSName("powerdns.com"), 300); + addRecordToLW(res, target3, QType::A, "192.0.2.4", DNSResourceRecord::ANSWER, 10); + + return LWResult::Result::Success; + }); + + const time_t now = sr->getNow().tv_sec; + + // we should only see a single record for the question we asked + vector ret; + int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); + BOOST_CHECK_EQUAL(res, RCode::NoError); + BOOST_REQUIRE_EQUAL(ret.size(), 2U); + BOOST_CHECK_EQUAL(sr->getValidationState(), vState::Secure); + BOOST_REQUIRE_EQUAL(QType(ret.at(0).d_type).toString(), QType(QType::A).toString()); + BOOST_CHECK_EQUAL(getRR(ret.at(0))->getCA().toString(), ComboAddress("192.0.2.2").toString()); + + // Also check the cache + const ComboAddress who; + vector cached; + BOOST_REQUIRE_GT(g_recCache->get(now, target, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_EQUAL(cached.size(), 1U); + BOOST_REQUIRE_EQUAL(QType(cached.at(0).d_type).toString(), QType(QType::A).toString()); + BOOST_CHECK_EQUAL(getRR(cached.at(0))->getCA().toString(), ComboAddress("192.0.2.2").toString()); + + // The cache should also have an authoritative record for the extra in-bailiwick record + BOOST_REQUIRE_GT(g_recCache->get(now, target2, QType(QType::A), true, &cached, who), 0); + BOOST_REQUIRE_EQUAL(cached.size(), 1U); + BOOST_REQUIRE_EQUAL(QType(cached.at(0).d_type).toString(), QType(QType::A).toString()); + BOOST_CHECK_EQUAL(getRR(cached.at(0))->getCA().toString(), ComboAddress("192.0.2.3").toString()); + + // But the out-of-bailiwick record should not be there + BOOST_REQUIRE_LT(g_recCache->get(now, target3, QType(QType::A), true, &cached, who), 0); +} + BOOST_AUTO_TEST_CASE(test_skip_opt_any) { std::unique_ptr sr; diff --git a/pdns/recursordist/test-syncres_cc4.cc b/pdns/recursordist/test-syncres_cc4.cc index e18b41f39ea5..23358d9bc8e7 100644 --- a/pdns/recursordist/test-syncres_cc4.cc +++ b/pdns/recursordist/test-syncres_cc4.cc @@ -927,8 +927,8 @@ BOOST_AUTO_TEST_CASE(test_dnssec_bogus_dnskey_doesnt_match_ds) /* === first without validation, then with (just-in-time validation) === */ /* clear the caches */ - g_recCache = std::unique_ptr(new MemRecursorCache()); - g_negCache = std::unique_ptr(new NegCache()); + g_recCache = std::make_unique(); + g_negCache = std::make_unique(); sr->setDNSSECValidationRequested(false); primeHints(); diff --git a/pdns/recursordist/test-syncres_cc6.cc b/pdns/recursordist/test-syncres_cc6.cc index 1a70714c79d0..c2c5d16a1d97 100644 --- a/pdns/recursordist/test-syncres_cc6.cc +++ b/pdns/recursordist/test-syncres_cc6.cc @@ -1120,7 +1120,7 @@ BOOST_AUTO_TEST_CASE(test_dnssec_secure_to_insecure_optout) addRecordToLW(res, domain, QType::SOA, "pdns-public-ns1.powerdns.com. pieter\\.lexis.powerdns.com. 2017032301 10800 3600 604800 3600", DNSResourceRecord::AUTHORITY, 3600); addRRSIG(keys, res->d_records, DNSName("com."), 300); /* closest encloser */ - addNSEC3UnhashedRecordToLW(DNSName("com."), DNSName("com."), DNSName("a.com.").toStringNoDot(), {QType::NS}, 600, res->d_records, 10, true); + addNSEC3UnhashedRecordToLW(DNSName("com."), DNSName("com."), DNSName("a.com.").toStringNoDot(), {QType::NS, QType::SOA}, 600, res->d_records, 10, true); addRRSIG(keys, res->d_records, DNSName("com."), 300); /* next closer */ addNSEC3UnhashedRecordToLW(DNSName("oowerdns.com."), DNSName("com."), DNSName("qowerdns.com.").toStringNoDot(), {QType::NS}, 600, res->d_records, 10, true); @@ -1165,7 +1165,7 @@ BOOST_AUTO_TEST_CASE(test_dnssec_secure_to_insecure_optout) addRecordToLW(res, domain, QType::NS, "ns1.powerdns.com.", DNSResourceRecord::AUTHORITY, 3600); /* no DS */ /* closest encloser */ - addNSEC3UnhashedRecordToLW(DNSName("com."), DNSName("com."), DNSName("a.com.").toStringNoDot(), {QType::NS}, 600, res->d_records, 10, true); + addNSEC3UnhashedRecordToLW(DNSName("com."), DNSName("com."), DNSName("a.com.").toStringNoDot(), {QType::NS, QType::SOA}, 600, res->d_records, 10, true); addRRSIG(keys, res->d_records, DNSName("com."), 300); /* next closer */ addNSEC3UnhashedRecordToLW(DNSName("oowerdns.com."), DNSName("com."), DNSName("qowerdns.com.").toStringNoDot(), {QType::NS}, 600, res->d_records, 10, true); @@ -1767,4 +1767,116 @@ BOOST_AUTO_TEST_CASE(test_dnssec_secure_to_broken_without_ds) BOOST_CHECK_EQUAL(queriesCount, 6U); } +BOOST_AUTO_TEST_CASE(test_dnssec_secure_to_broken_cname_ds) +{ + /* Test an Insecure domain that responds with a CNAME on a DS query */ + std::unique_ptr sr; + initSR(sr, true); + + setDNSSECValidation(sr, DNSSECMode::ValidateAll); + + primeHints(); + const DNSName target("www.sub.powerdns.com."); + const ComboAddress targetAddr("192.0.2.42"); + testkeysset_t keys, pdnskeys; + + auto luaconfsCopy = g_luaconfs.getCopy(); + luaconfsCopy.dsAnchors.clear(); + + // We have two set of keys as powerdns.com and sub.powerdns.com are Insecure but still have RRSIGS + generateKeyMaterial(g_rootdnsname, DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys, luaconfsCopy.dsAnchors); + generateKeyMaterial(DNSName("com."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys); + + generateKeyMaterial(DNSName("powerdns.com."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, pdnskeys); + generateKeyMaterial(DNSName("sub.powerdns.com."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, pdnskeys); + + g_luaconfs.setState(luaconfsCopy); + + size_t queriesCount = 0; + + sr->setAsyncCallback([target, targetAddr, &queriesCount, keys, pdnskeys](const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, LWResult* res, bool* chained) { + queriesCount++; + + if (type == QType::DS) { + // Return a signed CNAME on a DS query for powerdns.com and sub.powerdns.com + if (domain == DNSName("powerdns.com.") || domain == DNSName("sub.powerdns.com.")) { + setLWResult(res, 0, true, false, true); + addRecordToLW(res, domain, QType::CNAME, "some.name", DNSResourceRecord::ANSWER, 300); + addRRSIG(pdnskeys, res->d_records, domain, 300); + return LWResult::Result::Success; + } + else { + return genericDSAndDNSKEYHandler(res, domain, domain, type, keys); + } + } + else if (type == QType::DNSKEY) { + if (domain == DNSName("powerdns.com.") || domain == DNSName("sub.powerdns.com.")) { + return genericDSAndDNSKEYHandler(res, domain, domain, type, pdnskeys); + } + else { + return genericDSAndDNSKEYHandler(res, domain, domain, type, keys); + } + } + else { + if (isRootServer(ip)) { + setLWResult(res, 0, false, false, true); + addRecordToLW(res, "com.", QType::NS, "a.gtld-servers.com.", DNSResourceRecord::AUTHORITY, 3600); + addDS(DNSName("com."), 300, res->d_records, keys); + addRRSIG(keys, res->d_records, DNSName("."), 300); + addRecordToLW(res, "a.gtld-servers.com.", QType::A, "192.0.2.1", DNSResourceRecord::ADDITIONAL, 3600); + return LWResult::Result::Success; + } + else if (ip == ComboAddress("192.0.2.1:53")) { + if (domain == DNSName("com.")) { + setLWResult(res, 0, true, false, true); + addRecordToLW(res, DNSName("com."), QType::NS, "a.gtld-servers.com."); + addRRSIG(keys, res->d_records, DNSName("com."), 300); + addRecordToLW(res, "a.gtld-servers.com.", QType::A, "192.0.2.1", DNSResourceRecord::ADDITIONAL, 3600); + } + else { + setLWResult(res, 0, false, false, true); + addRecordToLW(res, DNSName("powerdns.com."), QType::NS, "ns1.powerdns.com.", DNSResourceRecord::AUTHORITY, 3600); + addNSECRecordToLW(DNSName("powerdns.com."), DNSName("z.powerdns.com."), {QType::NS, QType::RRSIG}, 600, res->d_records); + addRRSIG(keys, res->d_records, DNSName("com."), 300); + addRecordToLW(res, "ns1.powerdns.com.", QType::A, "192.0.2.2", DNSResourceRecord::ADDITIONAL, 3600); + } + return LWResult::Result::Success; + } + else if (ip == ComboAddress("192.0.2.2:53")) { + setLWResult(res, 0, false, false, true); + addRecordToLW(res, DNSName("sub.powerdns.com."), QType::NS, "ns1.sub.powerdns.com.", DNSResourceRecord::AUTHORITY, 3600); + addRRSIG(pdnskeys, res->d_records, DNSName("powerdns.com."), 300); + + addRecordToLW(res, "ns1.sub.powerdns.com.", QType::A, "192.0.2.3", DNSResourceRecord::ADDITIONAL, 3600); + return LWResult::Result::Success; + } + else if (ip == ComboAddress("192.0.2.3:53")) { + setLWResult(res, 0, true, false, true); + addRecordToLW(res, domain, QType::A, targetAddr.toString(), DNSResourceRecord::ANSWER, 3600); + addRRSIG(pdnskeys, res->d_records, DNSName("sub.powerdns.com."), 300); + return LWResult::Result::Success; + } + } + + return LWResult::Result::Timeout; + }); + + vector ret; + int res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); + BOOST_CHECK_EQUAL(res, RCode::NoError); + BOOST_CHECK_EQUAL(sr->getValidationState(), vState::Insecure); + BOOST_REQUIRE_EQUAL(ret.size(), 2U); + BOOST_CHECK(ret[0].d_type == QType::A); + BOOST_CHECK_EQUAL(queriesCount, 8U); + + /* again, to test the cache */ + ret.clear(); + res = sr->beginResolve(target, QType(QType::A), QClass::IN, ret); + BOOST_CHECK_EQUAL(res, RCode::NoError); + BOOST_CHECK_EQUAL(sr->getValidationState(), vState::Insecure); + BOOST_REQUIRE_EQUAL(ret.size(), 2U); + BOOST_CHECK(ret[0].d_type == QType::A); + BOOST_CHECK_EQUAL(queriesCount, 8U); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/recursordist/test-syncres_cc8.cc b/pdns/recursordist/test-syncres_cc8.cc index 509e4789e5cf..96a7330c5936 100644 --- a/pdns/recursordist/test-syncres_cc8.cc +++ b/pdns/recursordist/test-syncres_cc8.cc @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_denial_nowrap) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a.example.org."), QType::NSEC)] = pair; /* add wildcard denial */ recordContents.clear(); @@ -44,7 +44,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_denial_nowrap) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(DNSName("example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("example.org."), QType::NSEC)] = pair; dState denialState = getDenial(denialMap, DNSName("b.example.org."), QType::A, false, false); BOOST_CHECK_EQUAL(denialState, dState::NXDOMAIN); @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_denial_wrap_case_1) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("z.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("z.example.org."), QType::NSEC)] = pair; dState denialState = getDenial(denialMap, DNSName("a.example.org."), QType::A, false, false); BOOST_CHECK_EQUAL(denialState, dState::NXDOMAIN); @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_denial_wrap_case_2) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("y.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("y.example.org."), QType::NSEC)] = pair; dState denialState = getDenial(denialMap, DNSName("z.example.org."), QType::A, false, false); BOOST_CHECK_EQUAL(denialState, dState::NXDOMAIN); @@ -152,7 +152,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_denial_only_one_nsec) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a.example.org."), QType::NSEC)] = pair; dState denialState = getDenial(denialMap, DNSName("b.example.org."), QType::A, false, false); BOOST_CHECK_EQUAL(denialState, dState::NXDOMAIN); @@ -188,7 +188,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_root_nxd_denial) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a."), QType::NSEC)] = pair; /* add wildcard denial */ recordContents.clear(); @@ -201,7 +201,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_root_nxd_denial) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(DNSName("."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("."), QType::NSEC)] = pair; dState denialState = getDenial(denialMap, DNSName("b."), QType::A, false, false); BOOST_CHECK_EQUAL(denialState, dState::NXDOMAIN); @@ -223,7 +223,8 @@ BOOST_AUTO_TEST_CASE(test_nsec_ancestor_nxqtype_denial) The RRSIG from "." denies the existence of any type except NS at a. However since it's an ancestor delegation NSEC (NS bit set, SOA bit clear, signer field that is shorter than the owner name of the NSEC RR) it can't - be used to deny anything except the whole name or a DS. + be used to deny anything except the whole name (which does not make sense here) + or a DS. */ addNSECRecordToLW(DNSName("a."), DNSName("b."), {QType::NS}, 600, records); recordContents.insert(records.at(0).d_content); @@ -235,7 +236,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_ancestor_nxqtype_denial) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a."), QType::NSEC)] = pair; /* RFC 6840 section 4.1 "Clarifications on Nonexistence Proofs": Ancestor delegation NSEC or NSEC3 RRs MUST NOT be used to assume @@ -244,7 +245,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_ancestor_nxqtype_denial) owner name regardless of type. */ - dState denialState = getDenial(denialMap, DNSName("a."), QType::A, false, false); + dState denialState = getDenial(denialMap, DNSName("a."), QType::A, false, true); /* no data means the qname/qtype is not denied, because an ancestor delegation NSEC can only deny the DS */ BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); @@ -257,6 +258,38 @@ BOOST_AUTO_TEST_CASE(test_nsec_ancestor_nxqtype_denial) BOOST_CHECK_EQUAL(denialState, dState::NXQTYPE); } +BOOST_AUTO_TEST_CASE(test_nsec_ds_denial_from_child) +{ + initSR(); + + testkeysset_t keys; + generateKeyMaterial(DNSName("org."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys); + generateKeyMaterial(DNSName("example.org."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys); + + vector records; + + sortedRecords_t recordContents; + vector> signatureContents; + + addNSECRecordToLW(DNSName("example.org."), DNSName("a.example.org"), {QType::A, QType::TXT, QType::RRSIG, QType::NSEC}, 600, records); + recordContents.insert(records.at(0).d_content); + addRRSIG(keys, records, DNSName("example.org."), 300); + signatureContents.push_back(getRR(records.at(1))); + records.clear(); + + ContentSigPair pair; + pair.records = recordContents; + pair.signatures = signatureContents; + cspmap_t denialMap; + denialMap[std::pair(DNSName("example.org."), QType::NSEC)] = pair; + + /* check that this NSEC from the child zone can deny a AAAA at the apex */ + BOOST_CHECK_EQUAL(getDenial(denialMap, DNSName("example.org."), QType::AAAA, false, true, true), dState::NXQTYPE); + + /* but not that the DS does not exist, since we need the parent for that */ + BOOST_CHECK_EQUAL(getDenial(denialMap, DNSName("example.org."), QType::DS, false, true, true), dState::NODENIAL); +} + BOOST_AUTO_TEST_CASE(test_nsec_insecure_delegation_denial) { initSR(); @@ -291,7 +324,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_insecure_delegation_denial) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a."), QType::NSEC)] = pair; /* Insecure because the NS is not set, so while it does denies the DS, it can't prove an insecure delegation */ @@ -299,6 +332,46 @@ BOOST_AUTO_TEST_CASE(test_nsec_insecure_delegation_denial) BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); } +BOOST_AUTO_TEST_CASE(test_nsec_insecure_delegation_denial_soa) +{ + initSR(); + + testkeysset_t keys; + generateKeyMaterial(DNSName("."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys); + + vector records; + + sortedRecords_t recordContents; + vector> signatureContents; + + /* + * RFC 5155 section 8.9: + * If there is an NSEC3 RR present in the response that matches the + * delegation name, then the validator MUST ensure that the NS bit is + * set and that the DS bit is not set in the Type Bit Maps field of the + * NSEC3 RR. + */ + /* + The RRSIG from "." denies the existence of any type at a except NS and SOA. + NS has to be set since it is proving an insecure delegation, but SOA should NOT! + */ + addNSECRecordToLW(DNSName("a."), DNSName("b."), {QType::NS, QType::SOA}, 600, records); + recordContents.insert(records.at(0).d_content); + addRRSIG(keys, records, DNSName("."), 300); + signatureContents.push_back(getRR(records.at(1))); + records.clear(); + + ContentSigPair pair; + pair.records = recordContents; + pair.signatures = signatureContents; + cspmap_t denialMap; + denialMap[std::pair(DNSName("a."), QType::NSEC)] = pair; + + /* Insecure because both NS and SOA are set, so this is not a proper delegation */ + dState denialState = getDenial(denialMap, DNSName("a."), QType::DS, true, true); + BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); +} + BOOST_AUTO_TEST_CASE(test_nsec_nxqtype_cname) { initSR(); @@ -321,13 +394,43 @@ BOOST_AUTO_TEST_CASE(test_nsec_nxqtype_cname) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a.powerdns.com."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a.powerdns.com."), QType::NSEC)] = pair; /* this NSEC is not valid to deny a.powerdns.com|A since it states that a CNAME exists */ dState denialState = getDenial(denialMap, DNSName("a.powerdns.com."), QType::A, true, true); BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); } +BOOST_AUTO_TEST_CASE(test_nsec3_nxqtype_ds) +{ + initSR(); + + testkeysset_t keys; + generateKeyMaterial(DNSName("powerdns.com."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys); + + vector records; + + sortedRecords_t recordContents; + vector> signatureContents; + + addNSEC3UnhashedRecordToLW(DNSName("powerdns.com."), DNSName("powerdns.com."), "whatever", {QType::A}, 600, records); + recordContents.insert(records.at(0).d_content); + addRRSIG(keys, records, DNSName("powerdns.com."), 300); + signatureContents.push_back(getRR(records.at(1))); + + ContentSigPair pair; + pair.records = recordContents; + pair.signatures = signatureContents; + cspmap_t denialMap; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; + records.clear(); + + /* this NSEC3 is not valid to deny the DS since it is from the child zone */ + BOOST_CHECK_EQUAL(getDenial(denialMap, DNSName("powerdns.com."), QType::DS, false, true), dState::NODENIAL); + /* AAAA should be fine, though */ + BOOST_CHECK_EQUAL(getDenial(denialMap, DNSName("powerdns.com."), QType::AAAA, false, true), dState::NXQTYPE); +} + BOOST_AUTO_TEST_CASE(test_nsec3_nxqtype_cname) { initSR(); @@ -349,7 +452,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_nxqtype_cname) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; records.clear(); /* this NSEC3 is not valid to deny a.powerdns.com|A since it states that a CNAME exists */ @@ -379,7 +482,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_nxdomain_denial_missing_wildcard) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a.powerdns.com."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a.powerdns.com."), QType::NSEC)] = pair; dState denialState = getDenial(denialMap, DNSName("b.powerdns.com."), QType::A, false, false); BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); @@ -406,7 +509,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_nxdomain_denial_missing_wildcard) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* Add NSEC3 for the closest encloser */ recordContents.clear(); @@ -419,7 +522,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_nxdomain_denial_missing_wildcard) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; dState denialState = getDenial(denialMap, DNSName("a.powerdns.com."), QType::A, false, false); BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); @@ -448,7 +551,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_expanded_wildcard_proof) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a.example.org."), QType::NSEC)] = pair; /* This is an expanded wildcard proof, meaning that it does prove that the exact name does not exist so the wildcard can apply */ @@ -479,7 +582,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_wildcard_with_cname) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a.example.org."), QType::NSEC)] = pair; /* add a NSEC proving that a wildcard exists, without a CNAME type */ recordContents.clear(); @@ -492,7 +595,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_wildcard_with_cname) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(DNSName("*.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("*.example.org."), QType::NSEC)] = pair; /* A does exist at the wildcard, AAAA does not */ dState denialState = getDenial(denialMap, DNSName("b.example.org."), QType::A, false, true); @@ -512,7 +615,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_wildcard_with_cname) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(DNSName("*.example.org."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("*.example.org."), QType::NSEC)] = pair; /* A and AAAA do not exist but we have a CNAME so at the wildcard */ denialState = getDenial(denialMap, DNSName("b.example.org."), QType::A, false, true); @@ -544,7 +647,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_wildcard_with_cname) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* Add NSEC3 for the closest encloser */ recordContents.clear(); @@ -557,7 +660,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_wildcard_with_cname) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* add wildcard, without a CNAME type */ recordContents.clear(); @@ -570,7 +673,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_wildcard_with_cname) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* A does exist at the wildcard, AAAA does not */ dState denialState = getDenial(denialMap, DNSName("b.example.org."), QType::A, false, true); @@ -590,7 +693,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_wildcard_with_cname) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* A and AAAA do not exist but we have a CNAME so at the wildcard */ denialState = getDenial(denialMap, DNSName("b.example.org."), QType::A, false, true); @@ -622,7 +725,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_ent_denial) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(DNSName("a.powerdns.com."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName("a.powerdns.com."), QType::NSEC)] = pair; /* this NSEC is valid to prove a NXQTYPE at c.powerdns.com because it proves that it is an ENT */ @@ -648,7 +751,7 @@ BOOST_AUTO_TEST_CASE(test_nsec_ent_denial) records.clear(); pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(DNSName(").powerdns.com."), QType::NSEC)] = pair; + denialMap[std::pair(DNSName(").powerdns.com."), QType::NSEC)] = pair; denialState = getDenial(denialMap, DNSName("b.powerdns.com."), QType::A, true, false); BOOST_CHECK_EQUAL(denialState, dState::NXDOMAIN); @@ -686,7 +789,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_ancestor_nxqtype_denial) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; records.clear(); /* RFC 6840 section 4.1 "Clarifications on Nonexistence Proofs": @@ -716,7 +819,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_ancestor_nxqtype_denial) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* add wildcard denial */ recordContents.clear(); @@ -729,10 +832,14 @@ BOOST_AUTO_TEST_CASE(test_nsec3_ancestor_nxqtype_denial) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; denialState = getDenial(denialMap, DNSName("sub.a."), QType::A, false, true); BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); + + /* not even the DS! */ + denialState = getDenial(denialMap, DNSName("sub.a."), QType::DS, false, true); + BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); } BOOST_AUTO_TEST_CASE(test_nsec3_denial_too_many_iterations) @@ -757,7 +864,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_denial_too_many_iterations) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; records.clear(); dState denialState = getDenial(denialMap, DNSName("a."), QType::A, false, true); @@ -798,7 +905,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_insecure_delegation_denial) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; records.clear(); /* Insecure because the NS is not set, so while it does @@ -807,6 +914,46 @@ BOOST_AUTO_TEST_CASE(test_nsec3_insecure_delegation_denial) BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); } +BOOST_AUTO_TEST_CASE(test_nsec3_insecure_delegation_denial_soa) +{ + initSR(); + + testkeysset_t keys; + generateKeyMaterial(DNSName("."), DNSSECKeeper::ECDSA256, DNSSECKeeper::DIGEST_SHA256, keys); + + vector records; + + sortedRecords_t recordContents; + vector> signatureContents; + + /* + * RFC 5155 section 8.9: + * If there is an NSEC3 RR present in the response that matches the + * delegation name, then the validator MUST ensure that the NS bit is + * set and that the DS bit is not set in the Type Bit Maps field of the + * NSEC3 RR. + */ + /* + The RRSIG from "." denies the existence of any type at a except NS and SOA. + NS has to be set since it is proving an insecure delegation, but SOA should NOT! + */ + addNSEC3UnhashedRecordToLW(DNSName("a."), DNSName("."), "whatever", {QType::NS, QType::SOA}, 600, records); + recordContents.insert(records.at(0).d_content); + addRRSIG(keys, records, DNSName("."), 300); + signatureContents.push_back(getRR(records.at(1))); + + ContentSigPair pair; + pair.records = recordContents; + pair.signatures = signatureContents; + cspmap_t denialMap; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; + records.clear(); + + /* Insecure because both NS and SOA are set, so it is not a proper delegation */ + dState denialState = getDenial(denialMap, DNSName("a."), QType::DS, true, true); + BOOST_CHECK_EQUAL(denialState, dState::NODENIAL); +} + BOOST_AUTO_TEST_CASE(test_nsec3_ent_opt_out) { initSR(); @@ -840,7 +987,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_ent_opt_out) pair.records = recordContents; pair.signatures = signatureContents; cspmap_t denialMap; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* it can not be used to deny any RRs below that owner name either */ /* Add NSEC3 for the next closer */ @@ -854,7 +1001,7 @@ BOOST_AUTO_TEST_CASE(test_nsec3_ent_opt_out) pair.records = recordContents; pair.signatures = signatureContents; - denialMap[std::make_pair(records.at(0).d_name, records.at(0).d_type)] = pair; + denialMap[std::pair(records.at(0).d_name, records.at(0).d_type)] = pair; /* Insecure because the opt-out bit is set */ dState denialState = getDenial(denialMap, DNSName("ent.was.here."), QType::A, false, true); diff --git a/pdns/recursordist/test-syncres_cc9.cc b/pdns/recursordist/test-syncres_cc9.cc index d296f25bf4e5..48094541e53c 100644 --- a/pdns/recursordist/test-syncres_cc9.cc +++ b/pdns/recursordist/test-syncres_cc9.cc @@ -819,9 +819,9 @@ BOOST_AUTO_TEST_CASE(test_getDSRecords_multialgo_all_sha) dsmap_t ds; auto state = sr->getDSRecords(target, ds, false, 0, false); BOOST_CHECK_EQUAL(state, vState::Secure); - BOOST_REQUIRE_EQUAL(ds.size(), 1U); + BOOST_REQUIRE_EQUAL(ds.size(), 2U); for (const auto& i : ds) { - BOOST_CHECK_EQUAL(i.d_digesttype, DNSSECKeeper::DIGEST_SHA384); + BOOST_CHECK(i.d_digesttype == DNSSECKeeper::DIGEST_SHA384 || i.d_digesttype == DNSSECKeeper::DIGEST_SHA256); } } diff --git a/pdns/recursordist/testrunner.cc b/pdns/recursordist/testrunner.cc index 0dc784936c04..6d9b09374625 100644 --- a/pdns/recursordist/testrunner.cc +++ b/pdns/recursordist/testrunner.cc @@ -28,9 +28,55 @@ #include #include +#include +#include "logger.hh" +#include "logging.hh" + +static std::string s_timestampFormat = "%s"; + +static const char* toTimestampStringMilli(const struct timeval& tv, char* buf, size_t sz) + +{ + struct tm tm; + size_t len = strftime(buf, sz, s_timestampFormat.c_str(), localtime_r(&tv.tv_sec, &tm)); + if (len == 0) { + len = snprintf(buf, sz, "%lld", static_cast(tv.tv_sec)); + } + + snprintf(buf + len, sz - len, ".%03ld", static_cast(tv.tv_usec) / 1000); + return buf; +} + +static void loggerBackend(const Logging::Entry& entry) +{ + static thread_local std::stringstream buf; + + buf.str(""); + buf << "msg=" << std::quoted(entry.message); + if (entry.error) { + buf << " oserror=" << std::quoted(entry.error.get()); + } + + if (entry.name) { + buf << " subsystem=" << std::quoted(entry.name.get()); + } + buf << " level=" << entry.level; + if (entry.d_priority) { + buf << " prio=" << static_cast(entry.d_priority); + } + char timebuf[64]; + buf << " ts=" << std::quoted(toTimestampStringMilli(entry.d_timestamp, timebuf, sizeof(timebuf))); + for (auto const& v : entry.values) { + buf << " "; + buf << v.first << "=" << std::quoted(v.second); + } + Logger::Urgency u = entry.d_priority ? Logger::Urgency(entry.d_priority) : Logger::Info; + g_log << u << buf.str() << endl; +} static bool init_unit_test() { + g_slog = Logging::Logger::create(loggerBackend); reportAllTypes(); return true; } diff --git a/pdns/reczones.cc b/pdns/reczones.cc index a5430c85157e..384c7661f94e 100644 --- a/pdns/reczones.cc +++ b/pdns/reczones.cc @@ -53,7 +53,12 @@ bool primeHints(time_t ignored) time_t now = time(nullptr); - if (::arg()["hint-file"].empty()) { + const string hintfile = ::arg()["hint-file"]; + if (hintfile == "no") { + g_log << Logger::Debug << "Priming root disabled by hint-file=no" << endl; + return true; + } + if (hintfile.empty()) { DNSRecord arr, aaaarr, nsrr; nsrr.d_name = g_rootdnsname; arr.d_type = QType::A; @@ -97,8 +102,9 @@ bool primeHints(time_t ignored) } } else { - ZoneParserTNG zpt(::arg()["hint-file"]); + ZoneParserTNG zpt(hintfile); zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps")); + zpt.setMaxIncludes(::arg().asNum("max-include-depth")); DNSResourceRecord rr; set seenNS; set seenA; @@ -167,16 +173,14 @@ bool primeHints(time_t ignored) // servers are authoritative for root-servers.net, and some // implementations reply not with a delegation on a root-servers.net // DS query, but with a NODATA response (the domain is unsigned). -void primeRootNSZones(bool dnssecmode, unsigned int depth) +void primeRootNSZones(DNSSECMode mode, unsigned int depth) { struct timeval now; gettimeofday(&now, 0); SyncRes sr(now); - if (dnssecmode) { - sr.setDoDNSSEC(true); - sr.setDNSSECValidationRequested(true); - } + sr.setDoDNSSEC(mode != DNSSECMode::Off); + sr.setDNSSECValidationRequested(mode != DNSSECMode::Off && mode != DNSSECMode::ProcessNoValidate); // beginResolve() can yield to another mthread that could trigger t_rootNSZones updates, // so make a local copy @@ -294,7 +298,7 @@ static void* pleaseUseNewSDomainsMap(std::shared_ptr newma return 0; } -string reloadAuthAndForwards() +string reloadZoneConfiguration() { std::shared_ptr original = SyncRes::getDomainMap(); @@ -312,6 +316,8 @@ string reloadAuthAndForwards() ::arg().preParseFile(configname.c_str(), "forward-zones-file"); ::arg().preParseFile(configname.c_str(), "forward-zones-recurse"); ::arg().preParseFile(configname.c_str(), "auth-zones"); + ::arg().preParseFile(configname.c_str(), "allow-notify-for"); + ::arg().preParseFile(configname.c_str(), "allow-notify-for-file"); ::arg().preParseFile(configname.c_str(), "export-etc-hosts", "off"); ::arg().preParseFile(configname.c_str(), "serve-rfc1918"); ::arg().preParseFile(configname.c_str(), "include-dir"); @@ -327,6 +333,8 @@ string reloadAuthAndForwards() ::arg().preParseFile(fn.c_str(), "forward-zones-file", ::arg()["forward-zones-file"]); ::arg().preParseFile(fn.c_str(), "forward-zones-recurse", ::arg()["forward-zones-recurse"]); ::arg().preParseFile(fn.c_str(), "auth-zones", ::arg()["auth-zones"]); + ::arg().preParseFile(fn.c_str(), "allow-notify-for", ::arg()["allow-notify-for"]); + ::arg().preParseFile(fn.c_str(), "allow-notify-for-file", ::arg()["allow-notify-for-file"]); ::arg().preParseFile(fn.c_str(), "export-etc-hosts", ::arg()["export-etc-hosts"]); ::arg().preParseFile(fn.c_str(), "serve-rfc1918", ::arg()["serve-rfc1918"]); } @@ -335,10 +343,12 @@ string reloadAuthAndForwards() ::arg().preParse(g_argc, g_argv, "forward-zones-file"); ::arg().preParse(g_argc, g_argv, "forward-zones-recurse"); ::arg().preParse(g_argc, g_argv, "auth-zones"); + ::arg().preParse(g_argc, g_argv, "allow-notify-for"); + ::arg().preParse(g_argc, g_argv, "allow-notify-for-file"); ::arg().preParse(g_argc, g_argv, "export-etc-hosts"); ::arg().preParse(g_argc, g_argv, "serve-rfc1918"); - std::shared_ptr newDomainMap = parseAuthAndForwards(); + auto [newDomainMap, newNotifySet] = parseZoneConfiguration(); // purge both original and new names std::set oldAndNewDomains; @@ -353,12 +363,14 @@ string reloadAuthAndForwards() } for (const auto& i : oldAndNewDomains) { - g_recCache->doWipeCache(i, true, 0xffff); - broadcastAccFunction([&] { return pleaseWipePacketCache(i, true, 0xffff); }); - g_negCache->wipe(i, true); + wipeCaches(i, true, 0xffff); } - broadcastFunction([=] { return pleaseUseNewSDomainsMap(newDomainMap); }); + // these explicitly-named captures should not be necessary, as lambda + // capture of tuple-like structured bindings is permitted, but some + // compilers still don't allow it + broadcastFunction([dm = newDomainMap] { return pleaseUseNewSDomainsMap(dm); }); + broadcastFunction([ns = newNotifySet] { return pleaseSupplantAllowNotifyFor(ns); }); return "ok\n"; } catch (std::exception& e) { @@ -373,12 +385,13 @@ string reloadAuthAndForwards() return "reloading failed, see log\n"; } -std::shared_ptr parseAuthAndForwards() +std::tuple, std::shared_ptr> parseZoneConfiguration() { TXTRecordContent::report(); OPTRecordContent::report(); auto newMap = std::make_shared(); + auto newSet = std::make_shared(); typedef vector parts_t; parts_t parts; @@ -399,6 +412,7 @@ std::shared_ptr parseAuthAndForwards() g_log << Logger::Error << "Parsing authoritative data for zone '" << headers.first << "' from file '" << headers.second << "'" << endl; ZoneParserTNG zpt(headers.second, DNSName(headers.first)); zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps")); + zpt.setMaxIncludes(::arg().asNum("max-include-depth")); DNSResourceRecord rr; DNSRecord dr; while (zpt.get(rr)) { @@ -439,7 +453,6 @@ std::shared_ptr parseAuthAndForwards() if (!::arg()["forward-zones-file"].empty()) { g_log << Logger::Warning << "Reading zone forwarding information from '" << ::arg()["forward-zones-file"] << "'" << endl; - SyncRes::AuthDomain ad; auto fp = std::unique_ptr(fopen(::arg()["forward-zones-file"].c_str(), "r"), fclose); if (!fp) { throw PDNSException("Error opening forward-zones-file '" + ::arg()["forward-zones-file"] + "': " + stringerror()); @@ -449,6 +462,7 @@ std::shared_ptr parseAuthAndForwards() int linenum = 0; uint64_t before = newMap->size(); while (linenum++, stringfgets(fp.get(), line)) { + SyncRes::AuthDomain ad; boost::trim(line); if (line[0] == '#') // Comment line, skip to the next line continue; @@ -457,15 +471,27 @@ std::shared_ptr parseAuthAndForwards() instructions = splitField(instructions, '#').first; // Remove EOL comments boost::trim(domain); boost::trim(instructions); - if (domain.empty() && instructions.empty()) { // empty line - continue; + if (domain.empty()) { + if (instructions.empty()) { // empty line + continue; + } + throw PDNSException("Error parsing line " + std::to_string(linenum) + " of " + ::arg()["forward-zones-file"]); } - if (boost::starts_with(domain, "+")) { - domain = domain.c_str() + 1; - ad.d_rdForward = true; + + bool allowNotifyFor = false; + + for (; !domain.empty(); domain.erase(0, 1)) { + switch (domain[0]) { + case '+': + ad.d_rdForward = true; + continue; + case '^': + allowNotifyFor = true; + continue; + } + break; } - else - ad.d_rdForward = false; + if (domain.empty()) { throw PDNSException("Error parsing line " + std::to_string(linenum) + " of " + ::arg()["forward-zones-file"]); } @@ -479,6 +505,9 @@ std::shared_ptr parseAuthAndForwards() ad.d_name = DNSName(domain); (*newMap)[ad.d_name] = ad; + if (allowNotifyFor) { + newSet->insert(ad.d_name); + } } g_log << Logger::Warning << "Done parsing " << newMap->size() - before << " forwarding instructions from file '" << ::arg()["forward-zones-file"] << "'" << endl; } @@ -520,6 +549,7 @@ std::shared_ptr parseAuthAndForwards() } } } + if (::arg().mustDo("serve-rfc1918")) { g_log << Logger::Warning << "Inserting rfc 1918 private space zones" << endl; parts.clear(); @@ -535,5 +565,30 @@ std::shared_ptr parseAuthAndForwards() makeIPToNamesZone(newMap, parts); } } - return newMap; + + parts.clear(); + stringtok(parts, ::arg()["allow-notify-for"], " ,\t\n\r"); + for (parts_t::const_iterator iter = parts.begin(); iter != parts.end(); ++iter) { + newSet->insert(DNSName(*iter)); + } + + if (auto anff = ::arg()["allow-notify-for-file"]; !anff.empty()) { + g_log << Logger::Warning << "Reading NOTIFY-allowed zones from '" << anff << "'" << endl; + auto fp = std::unique_ptr(fopen(anff.c_str(), "r"), fclose); + if (!fp) { + throw PDNSException("Error opening allow-notify-for-file '" + anff + "': " + stringerror()); + } + + string line; + uint64_t before = newSet->size(); + while (stringfgets(fp.get(), line)) { + boost::trim(line); + if (line[0] == '#') // Comment line, skip to the next line + continue; + newSet->insert(DNSName(line)); + } + g_log << Logger::Warning << "Done parsing " << newSet->size() - before << " NOTIFY-allowed zones from file '" << anff << "'" << endl; + } + + return {newMap, newSet}; } diff --git a/pdns/remote_logger.cc b/pdns/remote_logger.cc index 736091118fcf..6e41db647f4e 100644 --- a/pdns/remote_logger.cc +++ b/pdns/remote_logger.cc @@ -170,7 +170,7 @@ void RemoteLogger::queueData(const std::string& data) } runtime->d_writer.write(data); - ++d_queued; + ++d_processed; } void RemoteLogger::maintenanceThread() diff --git a/pdns/remote_logger.hh b/pdns/remote_logger.hh index c00e882f228f..04db3b53ad86 100644 --- a/pdns/remote_logger.hh +++ b/pdns/remote_logger.hh @@ -90,7 +90,7 @@ public: void queueData(const std::string& data) override; std::string toString() const override { - return d_remote.toStringWithPort() + " (" + std::to_string(d_queued) + " queued, " + std::to_string(d_drops) + " dropped)"; + return d_remote.toStringWithPort() + " (" + std::to_string(d_processed) + " processed, " + std::to_string(d_drops) + " dropped)"; } void stop() { @@ -109,7 +109,7 @@ private: ComboAddress d_remote; std::atomic d_drops{0}; - std::atomic d_queued{0}; + std::atomic d_processed{0}; uint16_t d_timeout; uint8_t d_reconnectWaitTime; std::atomic d_exiting{false}; diff --git a/pdns/resolver.cc b/pdns/resolver.cc index 5cde2e88f344..6377008645d0 100644 --- a/pdns/resolver.cc +++ b/pdns/resolver.cc @@ -228,7 +228,7 @@ namespace pdns { bool Resolver::tryGetSOASerial(DNSName *domain, ComboAddress* remote, uint32_t *theirSerial, uint32_t *theirInception, uint32_t *theirExpire, uint16_t* id) { - auto fds = std::unique_ptr(new struct pollfd[locals.size()]); + auto fds = std::make_unique(locals.size()); size_t i = 0, k; int sock; diff --git a/pdns/responsestats-auth.cc b/pdns/responsestats-auth.cc index 833cd025a3ca..775b086089ef 100644 --- a/pdns/responsestats-auth.cc +++ b/pdns/responsestats-auth.cc @@ -22,6 +22,9 @@ void ResponseStats::submitResponse(DNSPacket &p, bool udpOrTCP, bool last) const static AtomicCounter &tcpbytesanswered4=*S.getPointer("tcp4-answers-bytes"); static AtomicCounter &tcpbytesanswered6=*S.getPointer("tcp6-answers-bytes"); + ComboAddress accountremote = p.d_remote; + if (p.d_inner_remote) accountremote = *p.d_inner_remote; + if(p.d.aa) { if (p.d.rcode==RCode::NXDomain) { S.inc("nxdomain-packets"); @@ -30,13 +33,13 @@ void ResponseStats::submitResponse(DNSPacket &p, bool udpOrTCP, bool last) const } else if (p.d.rcode == RCode::Refused) { S.inc("unauth-packets"); S.ringAccount("unauth-queries", p.qdomain, p.qtype); - S.ringAccount("remotes-unauth",p.d_remote); + S.ringAccount("remotes-unauth", accountremote); } if (udpOrTCP) { // udp udpnumanswered++; udpbytesanswered+=buf.length(); - if(p.d_remote.sin4.sin_family==AF_INET) { + if(accountremote.sin4.sin_family==AF_INET) { udpnumanswered4++; udpbytesanswered4+=buf.length(); } else { @@ -45,14 +48,14 @@ void ResponseStats::submitResponse(DNSPacket &p, bool udpOrTCP, bool last) const } } else { //tcp tcpbytesanswered+=buf.length(); - if(p.d_remote.sin4.sin_family==AF_INET) { + if(accountremote.sin4.sin_family==AF_INET) { tcpbytesanswered4+=buf.length(); } else { tcpbytesanswered6+=buf.length(); } if(last) { tcpnumanswered++; - if(p.d_remote.sin4.sin_family==AF_INET) { + if(accountremote.sin4.sin_family==AF_INET) { tcpnumanswered4++; } else { tcpnumanswered6++; diff --git a/pdns/responsestats.hh b/pdns/responsestats.hh index 0d89ca88ac7b..cb24c284bab2 100644 --- a/pdns/responsestats.hh +++ b/pdns/responsestats.hh @@ -23,6 +23,7 @@ #include #include "histogram.hh" +#include "stat_t.hh" #include "dnspacket.hh" @@ -42,7 +43,7 @@ public: private: struct Counter { - mutable std::atomic value; + mutable pdns::stat_t value; }; std::array d_qtypecounters; diff --git a/pdns/rfc2136handler.cc b/pdns/rfc2136handler.cc index ed0543792c9b..ea985b35c05f 100644 --- a/pdns/rfc2136handler.cc +++ b/pdns/rfc2136handler.cc @@ -156,7 +156,7 @@ uint PacketHandler::performUpdate(const string &msgPrefix, const DNSRecord *rr, di->backend->replaceRRSet(di->id, oldRec->qname, oldRec->qtype, rrset); *updatedSerial = true; changedRecords++; - g_log<d_name<<"|"<d_name<<"|"< 0) { di->backend->replaceRRSet(di->id, rr->d_name, rrType, rrset); - g_log<d_name<<"|"<d_name<<"|"<d_name<<"|"<d_name<<"|"<d_ttl) { i.ttl = rr->d_ttl; updateTTL++; @@ -204,10 +206,10 @@ uint PacketHandler::performUpdate(const string &msgPrefix, const DNSRecord *rr, } if (updateTTL > 0) { di->backend->replaceRRSet(di->id, rr->d_name, rrType, rrset); - g_log<d_name<<"|"<d_name<<"|"<d_name<<"|"<d_name<<"|"<d_class == QClass::IN) { - rrSetKey_t key = make_pair(rr->d_name, QType(rr->d_type)); + rrSetKey_t key = {rr->d_name, QType(rr->d_type)}; rrVector_t *vec = &preReqRRsets[key]; vec->push_back(DNSResourceRecord::fromWire(*rr)); } diff --git a/pdns/rpzloader.cc b/pdns/rpzloader.cc index 67896eac1564..03d822812628 100644 --- a/pdns/rpzloader.cc +++ b/pdns/rpzloader.cc @@ -64,7 +64,7 @@ Netmask makeNetmaskFromRPZ(const DNSName& name) return Netmask(v6); } -static void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr zone, bool addOrRemove, boost::optional defpol, bool defpolOverrideLocal, uint32_t maxTTL) +static void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr zone, bool addOrRemove, const boost::optional& defpol, bool defpolOverrideLocal, uint32_t maxTTL) { static const DNSName drop("rpz-drop."), truncate("rpz-tcp-only."), noaction("rpz-passthru."); static const DNSName rpzClientIP("rpz-client-ip"), rpzIP("rpz-ip"), @@ -188,14 +188,16 @@ static void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr loadRPZFromServer(const shared_ptr& plogger, const ComboAddress& primary, const DNSName& zoneName, std::shared_ptr zone, boost::optional defpol, bool defpolOverrideLocal, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout) +static shared_ptr loadRPZFromServer(const shared_ptr& plogger, const ComboAddress& primary, const DNSName& zoneName, std::shared_ptr zone, const boost::optional& defpol, bool defpolOverrideLocal, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout) { auto logger = plogger->withValues("primary", Logging::Loggable(primary)); - logger->info(Logr::Info, "Loading RPZ from nameserver"); + SLOG(g_log<info(Logr::Info, "Loading RPZ from nameserver")); logger = logger->v(1); if(!tt.name.empty()) { - logger->info(Logr::Info, "Using TSIG key for authentication", "tsig_key_name", Logging::Loggable(tt.name), "tsig_key_algorithm", Logging::Loggable(tt.algo)); + SLOG(g_log<info(Logr::Info, "Using TSIG key for authentication", "tsig_key_name", Logging::Loggable(tt.name), "tsig_key_algorithm", Logging::Loggable(tt.algo))); } ComboAddress local(localAddress); @@ -230,20 +232,61 @@ static shared_ptr loadRPZFromServer(const shared_ptrinfo(Logr::Info, "RPZ load in progress", "nrecords", Logging::Loggable(nrecords)); + SLOG(g_log<info(Logr::Info, "RPZ load in progress", "nrecords", Logging::Loggable(nrecords))); last=time(0); } } - logger->info(Logr::Info, "RPZ load completed", "nrecords", Logging::Loggable(nrecords), "soa", Logging::Loggable(sr->getZoneRepresentation())); + SLOG(g_log<getZoneRepresentation()<info(Logr::Info, "RPZ load completed", "nrecords", Logging::Loggable(nrecords), "soa", Logging::Loggable(sr->getZoneRepresentation()))); return sr; } +static LockGuarded > > s_rpzStats; + +shared_ptr getRPZZoneStats(const std::string& zone) +{ + auto stats = s_rpzStats.lock(); + auto it = stats->find(zone); + if (it == stats->end()) { + auto stat = std::make_shared(); + (*stats)[zone] = stat; + return stat; + } + return it->second; +} + +static void incRPZFailedTransfers(const std::string& zone) +{ + auto stats = getRPZZoneStats(zone); + if (stats != nullptr) + stats->d_failedTransfers++; +} + +static void setRPZZoneNewState(const std::string& zone, uint32_t serial, uint64_t numberOfRecords, bool fromFile, bool wasAXFR) +{ + auto stats = getRPZZoneStats(zone); + if (stats == nullptr) { + return; + } + if (!fromFile) { + stats->d_successfulTransfers++; + if (wasAXFR) { + stats->d_fullTransfers++; + } + } + stats->d_lastUpdate = time(nullptr); + stats->d_serial = serial; + stats->d_numberOfRecords = numberOfRecords; +} + // this function is silent - you do the logging -std::shared_ptr loadRPZFromFile(const std::string& fname, std::shared_ptr zone, boost::optional defpol, bool defpolOverrideLocal, uint32_t maxTTL) +std::shared_ptr loadRPZFromFile(const std::string& fname, std::shared_ptr zone, const boost::optional& defpol, bool defpolOverrideLocal, uint32_t maxTTL) { shared_ptr sr = nullptr; ZoneParserTNG zpt(fname); zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps")); + zpt.setMaxIncludes(::arg().asNum("max-include-depth")); DNSResourceRecord drr; DNSName domain; while(zpt.get(drr)) { @@ -271,45 +314,11 @@ std::shared_ptr loadRPZFromFile(const std::string& fname, std: if (sr != nullptr) { zone->setRefresh(sr->d_st.refresh); + setRPZZoneNewState(zone->getName(), sr->d_st.serial, zone->size(), true, false); } return sr; } -static LockGuarded > > s_rpzStats; - -shared_ptr getRPZZoneStats(const std::string& zone) -{ - auto stats = s_rpzStats.lock(); - auto it = stats->find(zone); - if (it == stats->end()) { - auto stat = std::make_shared(); - (*stats)[zone] = stat; - return stat; - } - return it->second; -} - -static void incRPZFailedTransfers(const std::string& zone) -{ - auto stats = getRPZZoneStats(zone); - if (stats != nullptr) - stats->d_failedTransfers++; -} - -static void setRPZZoneNewState(const std::string& zone, uint32_t serial, uint64_t numberOfRecords, bool wasAXFR) -{ - auto stats = getRPZZoneStats(zone); - if (stats == nullptr) - return; - stats->d_successfulTransfers++; - if (wasAXFR) { - stats->d_fullTransfers++; - } - stats->d_lastUpdate = time(nullptr); - stats->d_serial = serial; - stats->d_numberOfRecords = numberOfRecords; -} - static bool dumpZoneToDisk(const shared_ptr& plogger, const DNSName& zoneName, const std::shared_ptr& newZone, const std::string& dumpZoneFileName) { auto logger = plogger->v(1); @@ -317,7 +326,8 @@ static bool dumpZoneToDisk(const shared_ptr& plogger, const DNSNam std::string temp = dumpZoneFileName + "XXXXXX"; int fd = mkstemp(&temp.at(0)); if (fd < 0) { - logger->error(Logr::Warning, errno, "Unable to create temporary file"); + SLOG(g_log<error(Logr::Warning, errno, "Unable to create temporary file")); return false; } @@ -325,7 +335,8 @@ static bool dumpZoneToDisk(const shared_ptr& plogger, const DNSNam if (!fp) { int err = errno; close(fd); - logger->error(Logr::Warning, err, "Unable to open file pointer"); + SLOG(g_log<error(Logr::Warning, err, "Unable to open file pointer")); return false; } fd = -1; @@ -334,34 +345,39 @@ static bool dumpZoneToDisk(const shared_ptr& plogger, const DNSNam newZone->dump(fp.get()); } catch(const std::exception& e) { - logger->error(Logr::Warning, e.what(), "Error while dumping the content of the RPZ"); + SLOG(g_log<error(Logr::Warning, e.what(), "Error while dumping the content of the RPZ")); return false; } if (fflush(fp.get()) != 0) { - logger->error(Logr::Warning, errno, "Error while flushing the content of the RPZ"); + SLOG(g_log<error(Logr::Warning, errno, "Error while flushing the content of the RPZ")); return false; } if (fsync(fileno(fp.get())) != 0) { - logger->error(Logr::Warning, errno, "Error while syncing the content of the RPZ"); + SLOG(g_log<error(Logr::Warning, errno, "Error while syncing the content of the RPZ")); return false; } if (fclose(fp.release()) != 0) { - logger->error(Logr::Warning, errno, "Error while writing the content of the RPZ"); + SLOG(g_log<error(Logr::Warning, errno, "Error while writing the content of the RPZ")); return false; } if (rename(temp.c_str(), dumpZoneFileName.c_str()) != 0) { - logger->error(Logr::Warning, errno, "Error while moving the content of the RPZ", "destination_file", Logging::Loggable(dumpZoneFileName)); + SLOG(g_log<error(Logr::Warning, errno, "Error while moving the content of the RPZ", "destination_file", Logging::Loggable(dumpZoneFileName))); return false; } return true; } -void RPZIXFRTracker(const std::vector& primaries, boost::optional defpol, bool defpolOverrideLocal, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, const uint32_t refreshFromConf, std::shared_ptr sr, std::string dumpZoneFileName, uint64_t configGeneration) +void RPZIXFRTracker(const std::vector& primaries, const boost::optional& defpol, bool defpolOverrideLocal, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, const uint32_t refreshFromConf, std::shared_ptr sr, const std::string& dumpZoneFileName, uint64_t configGeneration) { setThreadName("pdns-r/RPZIXFR"); bool isPreloaded = sr != nullptr; @@ -372,14 +388,15 @@ void RPZIXFRTracker(const std::vector& primaries, boost::optional< /* we can _never_ modify this zone directly, we need to do a full copy then replace the existing zone */ std::shared_ptr oldZone = luaconfsLocal->dfe.getZone(zoneIdx); if (!oldZone) { - logger->error(Logr::Error, "Unable to retrieve RPZ zone from configuration", "index", Logging::Loggable(zoneIdx)); + SLOG(g_log<error(Logr::Error, "Unable to retrieve RPZ zone from configuration", "index", Logging::Loggable(zoneIdx))); return; } // If oldZone failed to load its getRefresh() returns 0, protect against that uint32_t refresh = std::max(refreshFromConf ? refreshFromConf : oldZone->getRefresh(), 10U); DNSName zoneName = oldZone->getDomain(); - std::string polName = oldZone->getName().empty() ? oldZone->getName() : zoneName.toString(); + std::string polName = !oldZone->getName().empty() ? oldZone->getName() : zoneName.toStringNoDot(); // Now that we know the name, set it in the logger logger = logger->withValues("zone", Logging::Loggable(zoneName)); @@ -395,7 +412,7 @@ void RPZIXFRTracker(const std::vector& primaries, boost::optional< newZone->setSerial(sr->d_st.serial); newZone->setRefresh(sr->d_st.refresh); refresh = std::max(refreshFromConf ? refreshFromConf : newZone->getRefresh(), 1U); - setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), true); + setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), false, true); g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) { lci.dfe.setZone(zoneIdx, newZone); @@ -409,11 +426,13 @@ void RPZIXFRTracker(const std::vector& primaries, boost::optional< break; } catch(const std::exception& e) { - logger->info(Logr::Warning, "Unable to load RPZ zone, will retry", "from", Logging::Loggable(primary), "exception", Logging::Loggable(e.what()), "refresh", Logging::Loggable(refresh)); + SLOG(g_log<info(Logr::Warning, "Unable to load RPZ zone, will retry", "from", Logging::Loggable(primary), "exception", Logging::Loggable(e.what()), "refresh", Logging::Loggable(refresh))); incRPZFailedTransfers(polName); } catch(const PDNSException& e) { - logger->info(Logr::Warning, "Unable to load RPZ zone, will retry", "from", Logging::Loggable(primary), "exception", Logging::Loggable(e.reason), "refresh", Logging::Loggable(refresh)); + SLOG(g_log<info(Logr::Warning, "Unable to load RPZ zone, will retry", "from", Logging::Loggable(primary), "exception", Logging::Loggable(e.reason), "refresh", Logging::Loggable(refresh))); incRPZFailedTransfers(polName); } } @@ -545,7 +564,7 @@ void RPZIXFRTracker(const std::vector& primaries, boost::optional< g_log<d_st.serial<setSerial(sr->d_st.serial); newZone->setRefresh(sr->d_st.refresh); - setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), fullUpdate); + setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), false, fullUpdate); /* we need to replace the existing zone with the new one, but we don't want to touch anything else, especially other zones, diff --git a/pdns/rpzloader.hh b/pdns/rpzloader.hh index 85e10d30a435..304ab5ab081f 100644 --- a/pdns/rpzloader.hh +++ b/pdns/rpzloader.hh @@ -26,8 +26,8 @@ extern bool g_logRPZChanges; -std::shared_ptr loadRPZFromFile(const std::string& fname, std::shared_ptr zone, boost::optional defpol, bool defpolOverrideLocal, uint32_t maxTTL); -void RPZIXFRTracker(const std::vector& primaries, boost::optional defpol, bool defpolOverrideLocal, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, const uint32_t reloadFromConf, shared_ptr sr, std::string dumpZoneFileName, uint64_t configGeneration); +std::shared_ptr loadRPZFromFile(const std::string& fname, std::shared_ptr zone, const boost::optional& defpol, bool defpolOverrideLocal, uint32_t maxTTL); +void RPZIXFRTracker(const std::vector& primaries, const boost::optional& defpol, bool defpolOverrideLocal, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, const uint32_t reloadFromConf, shared_ptr sr, const std::string& dumpZoneFileName, uint64_t configGeneration); struct rpzStats { diff --git a/pdns/saxfr.cc b/pdns/saxfr.cc index a7517804f0c5..ef0761e5edd1 100644 --- a/pdns/saxfr.cc +++ b/pdns/saxfr.cc @@ -112,7 +112,7 @@ try throw PDNSException("tcp read failed"); len=ntohs(len); - std::unique_ptr creply(new char[len]); + auto creply = std::make_unique(len); int n=0; int numread; while(nfirst.d_content->getZoneRepresentation(); } - records.push_back(make_pair(i->first.d_name,o.str())); + records.emplace_back(i->first.d_name, o.str()); DNSName shorter(i->first.d_name); do { diff --git a/pdns/sdig.cc b/pdns/sdig.cc index 12eb822bb3c8..6e9d77635b61 100644 --- a/pdns/sdig.cc +++ b/pdns/sdig.cc @@ -41,8 +41,8 @@ static void usage() "[dnssec] [ednssubnet SUBNET/MASK] [hidesoadetails] [hidettl] [recurse] [showflags] " "[tcp] [dot] [insecure] [fastOpen] [subjectName name] [caStore file] [tlsProvider openssl|gnutls] " "[xpf XPFDATA] [class CLASSNUM] " - "[proxy UDP(0)/TCP(1) SOURCE-IP-ADDRESS-AND-PORT DESTINATION-IP-ADDRESS-AND-PORT]" - "dumpluaraw" + "[proxy UDP(0)/TCP(1) SOURCE-IP-ADDRESS-AND-PORT DESTINATION-IP-ADDRESS-AND-PORT] " + "[dumpluaraw] [opcode OPNUM]" << endl; } @@ -60,9 +60,9 @@ static void fillPacket(vector& packet, const string& q, const string& t bool dnssec, const boost::optional ednsnm, bool recurse, uint16_t xpfcode, uint16_t xpfversion, uint64_t xpfproto, char* xpfsrc, char* xpfdst, - QClass qclass, uint16_t qid) + QClass qclass, uint8_t opcode, uint16_t qid) { - DNSPacketWriter pw(packet, DNSName(q), DNSRecordContent::TypeToNumber(t), qclass); + DNSPacketWriter pw(packet, DNSName(q), DNSRecordContent::TypeToNumber(t), qclass, opcode); if (dnssec || ednsnm || getenv("SDIGBUFSIZE")) { char* sbuf = getenv("SDIGBUFSIZE"); @@ -75,8 +75,7 @@ static void fillPacket(vector& packet, const string& q, const string& t if (ednsnm) { EDNSSubnetOpts eo; eo.source = *ednsnm; - opts.push_back( - make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(eo))); + opts.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(eo)); } pw.addOpt(bufsize, 0, dnssec ? EDNSOpts::DNSSECOK : 0, opts); @@ -212,6 +211,7 @@ try { uint16_t xpfcode = 0, xpfversion = 0, xpfproto = 0; char *xpfsrc = NULL, *xpfdst = NULL; QClass qclass = QClass::IN; + uint8_t opcode = 0; string proxyheader; string subjectName; string caStore; @@ -282,6 +282,13 @@ try { } qclass = atoi(argv[++i]); } + else if (strcmp(argv[i], "opcode") == 0) { + if (argc < i+2) { + cerr << "opcode needs an argument"< packet; s_expectedIDs.insert(0); fillPacket(packet, name, type, dnssec, ednsnm, recurse, xpfcode, xpfversion, - xpfproto, xpfsrc, xpfdst, qclass, 0); + xpfproto, xpfsrc, xpfdst, qclass, opcode, 0); MiniCurl mc; MiniCurl::MiniCurlHeaders mch; - mch.insert(std::make_pair("Content-Type", "application/dns-message")); - mch.insert(std::make_pair("Accept", "application/dns-message")); + mch.emplace("Content-Type", "application/dns-message"); + mch.emplace("Accept", "application/dns-message"); string question(packet.begin(), packet.end()); // FIXME: how do we use proxyheader here? reply = mc.postURL(argv[1], question, mch, timeout.tv_sec, fastOpen); @@ -422,7 +429,7 @@ try { vector packet; s_expectedIDs.insert(counter); fillPacket(packet, it.first, it.second, dnssec, ednsnm, recurse, xpfcode, - xpfversion, xpfproto, xpfsrc, xpfdst, qclass, counter); + xpfversion, xpfproto, xpfsrc, xpfdst, qclass, opcode, counter); counter++; // Prefer to do a single write, so that fastopen can send all the data on SYN @@ -453,7 +460,7 @@ try { vector packet; s_expectedIDs.insert(0); fillPacket(packet, name, type, dnssec, ednsnm, recurse, xpfcode, xpfversion, - xpfproto, xpfsrc, xpfdst, qclass, 0); + xpfproto, xpfsrc, xpfdst, qclass, opcode, 0); string question(packet.begin(), packet.end()); Socket sock(dest.sin4.sin_family, SOCK_DGRAM); question = proxyheader + question; diff --git a/pdns/secpoll-recursor.cc b/pdns/secpoll-recursor.cc index f5971558f11b..69618030f82c 100644 --- a/pdns/secpoll-recursor.cc +++ b/pdns/secpoll-recursor.cc @@ -10,7 +10,7 @@ #include "secpoll.hh" #include -#ifndef PACKAGEVERSION +#ifndef PACKAGEVERSION #define PACKAGEVERSION getPDNSVersion() #endif @@ -19,7 +19,7 @@ string g_security_message; void doSecPoll(time_t* last_secpoll) { - if(::arg()["security-poll-suffix"].empty()) + if (::arg()["security-poll-suffix"].empty()) return; string pkgv(PACKAGEVERSION); @@ -28,7 +28,7 @@ void doSecPoll(time_t* last_secpoll) /* update last_secpoll right now, even if it fails we don't want to retry right away and hammer the server */ - *last_secpoll=now.tv_sec; + *last_secpoll = now.tv_sec; SyncRes sr(now); if (g_dnssecmode != DNSSECMode::Off) { @@ -38,11 +38,11 @@ void doSecPoll(time_t* last_secpoll) vector ret; - string version = "recursor-" +pkgv; - string qstring(version.substr(0, 63)+ ".security-status."+::arg()["security-poll-suffix"]); + string version = "recursor-" + pkgv; + string qstring(version.substr(0, 63) + ".security-status." + ::arg()["security-poll-suffix"]); - if(*qstring.rbegin()!='.') - qstring+='.'; + if (*qstring.rbegin() != '.') + qstring += '.'; boost::replace_all(qstring, "+", "_"); boost::replace_all(qstring, "~", "_"); @@ -56,14 +56,14 @@ void doSecPoll(time_t* last_secpoll) } if (vStateIsBogus(state)) { - g_log< -void doSecPoll(time_t* ); +void doSecPoll(time_t*); extern uint32_t g_security_status; extern std::string g_security_message; diff --git a/pdns/signingpipe.cc b/pdns/signingpipe.cc index b859bae7ecd9..7ee3634f1082 100644 --- a/pdns/signingpipe.cc +++ b/pdns/signingpipe.cc @@ -32,7 +32,8 @@ int readn(int fd, void* buffer, unsigned int len) if(errno == EAGAIN || errno == EINTR) { if(pos==0) return -1; - waitForData(fd, -1); + // error handled later + (void)waitForData(fd, -1); continue; } unixDie("Reading from socket in Signing Pipe loop"); diff --git a/pdns/slavecommunicator.cc b/pdns/slavecommunicator.cc index 17d1ea33e2d6..79b0d3955097 100644 --- a/pdns/slavecommunicator.cc +++ b/pdns/slavecommunicator.cc @@ -50,20 +50,20 @@ void CommunicatorClass::addSuckRequest(const DNSName &domain, const ComboAddress& master, SuckRequest::RequestPriority priority, bool force) { - std::lock_guard l(d_lock); + auto data = d_data.lock(); SuckRequest sr; sr.domain = domain; sr.master = master; sr.force = force; sr.priorityAndOrder.first = priority; - sr.priorityAndOrder.second = d_sorthelper++; + sr.priorityAndOrder.second = data->d_sorthelper++; pair res; - res=d_suckdomains.insert(sr); + res = data->d_suckdomains.insert(sr); if(res.second) { d_suck_sem.post(); } else { - d_suckdomains.modify(res.first, [priorityAndOrder = sr.priorityAndOrder] (SuckRequest& so) { + data->d_suckdomains.modify(res.first, [priorityAndOrder = sr.priorityAndOrder] (SuckRequest& so) { if (priorityAndOrder.first < so.priorityAndOrder.first) { so.priorityAndOrder = priorityAndOrder; } @@ -307,11 +307,11 @@ static vector doAxfr(const ComboAddress& raddr, const DNSName void CommunicatorClass::suck(const DNSName &domain, const ComboAddress& remote, bool force) { { - std::lock_guard l(d_lock); - if(d_inprogress.count(domain)) { + auto data = d_data.lock(); + if (data->d_inprogress.count(domain)) { return; } - d_inprogress.insert(domain); + data->d_inprogress.insert(domain); } RemoveSentinel rs(domain, this); // this removes us from d_inprogress when we go out of scope @@ -649,17 +649,18 @@ void CommunicatorClass::suck(const DNSName &domain, const ComboAddress& remote, } catch(ResolverException &re) { { - std::lock_guard l(d_lock); + auto data = d_data.lock(); // The AXFR probably failed due to a problem on the master server. If SOA-checks against this master // still succeed, we would constantly try to AXFR the zone. To avoid this, we add the zone to the list of // failed slave-checks. This will suspend slave-checks (and subsequent AXFR) for this zone for some time. uint64_t newCount = 1; time_t now = time(nullptr); - const auto failedEntry = d_failedSlaveRefresh.find(domain); - if (failedEntry != d_failedSlaveRefresh.end()) - newCount = d_failedSlaveRefresh[domain].first + 1; + const auto failedEntry = data->d_failedSlaveRefresh.find(domain); + if (failedEntry != data->d_failedSlaveRefresh.end()) { + newCount = data->d_failedSlaveRefresh[domain].first + 1; + } time_t nextCheck = now + std::min(newCount * d_tickinterval, (uint64_t)::arg().asNum("default-ttl")); - d_failedSlaveRefresh[domain] = {newCount, nextCheck}; + data->d_failedSlaveRefresh[domain] = {newCount, nextCheck}; g_log< l(d_lock); + auto data = d_data.lock(); DomainInfo ours = di; ours.backend = nullptr; @@ -755,17 +756,18 @@ void CommunicatorClass::addSlaveCheckRequest(const DomainInfo& di, const ComboAd break; } } - d_tocheck.erase(di); - d_tocheck.insert(ours); + data->d_tocheck.erase(di); + data->d_tocheck.insert(ours); d_any_sem.post(); // kick the loop! } void CommunicatorClass::addTrySuperMasterRequest(const DNSPacket& p) { - std::lock_guard l(d_lock); const DNSPacket& ours = p; - if(d_potentialsupermasters.insert(ours).second) + auto data = d_data.lock(); + if (data->d_potentialsupermasters.insert(ours).second) { d_any_sem.post(); // kick the loop! + } } void CommunicatorClass::slaveRefresh(PacketHandler *P) @@ -776,33 +778,33 @@ void CommunicatorClass::slaveRefresh(PacketHandler *P) UeberBackend *B=P->getBackend(); vector rdomains; vector sdomains; - set trysuperdomains; + set trysuperdomains; { - std::lock_guard l(d_lock); + auto data = d_data.lock(); set requeue; - rdomains.reserve(d_tocheck.size()); - for(const auto& di: d_tocheck) { - if(d_inprogress.count(di.zone)) { + rdomains.reserve(data->d_tocheck.size()); + for (const auto& di: data->d_tocheck) { + if (data->d_inprogress.count(di.zone)) { g_log<d_failedSlaveRefresh.find(di.zone); + if (wasFailedDomain != data->d_failedSlaveRefresh.end()) { g_log<d_failedSlaveRefresh.erase(di.zone); } else { g_log<d_tocheck.swap(requeue); - trysuperdomains = d_potentialsupermasters; - d_potentialsupermasters.clear(); + trysuperdomains = std::move(data->d_potentialsupermasters); + data->d_potentialsupermasters.clear(); } for(const DNSPacket& dp : trysuperdomains) { @@ -818,13 +820,13 @@ void CommunicatorClass::slaveRefresh(PacketHandler *P) sdomains.reserve(rdomains.size()); DNSSECKeeper dk(B); // NOW HEAR THIS! This DK uses our B backend, so no interleaved access! { - std::lock_guard l(d_lock); - domains_by_name_t& nameindex=boost::multi_index::get(d_suckdomains); + auto data = d_data.lock(); + domains_by_name_t& nameindex=boost::multi_index::get(data->d_suckdomains); time_t now = time(nullptr); for(DomainInfo& di : rdomains) { - const auto failed = d_failedSlaveRefresh.find(di.zone); - if (failed != d_failedSlaveRefresh.end() && now < failed->second.second ) { + const auto failed = data->d_failedSlaveRefresh.find(di.zone); + if (failed != data->d_failedSlaveRefresh.end() && now < failed->second.second ) { // If the domain has failed before and the time before the next check has not expired, skip this domain g_log<second.second<d_inprogress.count(sr.domain)) { // this does continue; + } DomainNotificationInfo dni; dni.di=di; @@ -878,18 +881,18 @@ void CommunicatorClass::slaveRefresh(PacketHandler *P) } if(sdomains.empty()) { - if(d_slaveschanged) { - std::lock_guard l(d_lock); - g_log<d_suckdomains.size()<<" queued for AXFR already, "<d_inprogress.size()<<" in progress"< l(d_lock); + auto data = d_data.lock(); g_log<1 ? "s" : "")<<" need"<< (sdomains.size()>1 ? "" : "s")<< - " checking, "<d_suckdomains.size()<<" queued for AXFR"< l(d_lock); - const auto failedEntry = d_failedSlaveRefresh.find(di.zone); - if (failedEntry != d_failedSlaveRefresh.end()) - newCount = d_failedSlaveRefresh[di.zone].first + 1; + auto data = d_data.lock(); + const auto failedEntry = data->d_failedSlaveRefresh.find(di.zone); + if (failedEntry != data->d_failedSlaveRefresh.end()) + newCount = data->d_failedSlaveRefresh[di.zone].first + 1; time_t nextCheck = now + std::min(newCount * d_tickinterval, (uint64_t)::arg().asNum("default-ttl")); - d_failedSlaveRefresh[di.zone] = {newCount, nextCheck}; + data->d_failedSlaveRefresh[di.zone] = {newCount, nextCheck}; if (newCount == 1) { g_log< l(d_lock); - const auto wasFailedDomain = d_failedSlaveRefresh.find(di.zone); - if (wasFailedDomain != d_failedSlaveRefresh.end()) - d_failedSlaveRefresh.erase(di.zone); + auto data = d_data.lock(); + const auto wasFailedDomain = data->d_failedSlaveRefresh.find(di.zone); + if (wasFailedDomain != data->d_failedSlaveRefresh.end()) + data->d_failedSlaveRefresh.erase(di.zone); } bool hasSOA = false; @@ -1050,15 +1053,14 @@ void CommunicatorClass::slaveRefresh(PacketHandler *P) vector > CommunicatorClass::getSuckRequests() { vector > ret; - std::lock_guard l(d_lock); - ret.reserve(d_suckdomains.size()); - for (auto const &d : d_suckdomains) { - ret.push_back(make_pair(d.domain, d.master)); + auto data = d_data.lock(); + ret.reserve(data->d_suckdomains.size()); + for (auto const &d : data->d_suckdomains) { + ret.emplace_back(d.domain, d.master); } return ret; } size_t CommunicatorClass::getSuckRequestsWaiting() { - std::lock_guard l(d_lock); - return d_suckdomains.size(); + return d_data.lock()->d_suckdomains.size(); } diff --git a/pdns/sodiumsigners.cc b/pdns/sodiumsigners.cc index d33eb20ec37b..90ed36fb122d 100644 --- a/pdns/sodiumsigners.cc +++ b/pdns/sodiumsigners.cc @@ -58,10 +58,10 @@ DNSCryptoKeyEngine::storvector_t SodiumED25519DNSCryptoKeyEngine::convertToISCVe storvector_t storvector; string algorithm = "15 (ED25519)"; - storvector.push_back(make_pair("Algorithm", algorithm)); + storvector.emplace_back("Algorithm", algorithm); vector buffer; - storvector.push_back(make_pair("PrivateKey", string((char*)d_seckey, crypto_sign_ed25519_SEEDBYTES))); + storvector.emplace_back("PrivateKey", string((char*)d_seckey, crypto_sign_ed25519_SEEDBYTES)); return storvector; } @@ -79,7 +79,7 @@ void SodiumED25519DNSCryptoKeyEngine::fromISCMap(DNSKEYRecordContent& drc, std:: if (privateKey.length() != crypto_sign_ed25519_SEEDBYTES) throw runtime_error("Seed size mismatch in ISCMap, SodiumED25519 class"); - std::unique_ptr seed(new unsigned char[crypto_sign_ed25519_SEEDBYTES]); + auto seed = std::make_unique(crypto_sign_ed25519_SEEDBYTES); memcpy(seed.get(), privateKey.c_str(), crypto_sign_ed25519_SEEDBYTES); crypto_sign_ed25519_seed_keypair(d_pubkey, d_seckey, seed.get()); @@ -106,7 +106,7 @@ void SodiumED25519DNSCryptoKeyEngine::fromPublicKeyString(const std::string& inp std::string SodiumED25519DNSCryptoKeyEngine::sign(const std::string& msg) const { unsigned long long smlen = msg.length() + crypto_sign_ed25519_BYTES; - std::unique_ptr sm(new unsigned char[smlen]); + auto sm = std::make_unique(smlen); crypto_sign_ed25519(sm.get(), &smlen, (const unsigned char*)msg.c_str(), msg.length(), d_seckey); @@ -119,12 +119,12 @@ bool SodiumED25519DNSCryptoKeyEngine::verify(const std::string& msg, const std:: return false; unsigned long long smlen = msg.length() + crypto_sign_ed25519_BYTES; - std::unique_ptr sm(new unsigned char[smlen]); + auto sm = std::make_unique(smlen); memcpy(sm.get(), signature.c_str(), crypto_sign_ed25519_BYTES); memcpy(sm.get() + crypto_sign_ed25519_BYTES, msg.c_str(), msg.length()); - std::unique_ptr m(new unsigned char[smlen]); + auto m = std::make_unique(smlen); return crypto_sign_ed25519_open(m.get(), &smlen, sm.get(), smlen, d_pubkey) == 0; } diff --git a/pdns/speedtest.cc b/pdns/speedtest.cc index 34715eac4c76..652b7ecf0982 100644 --- a/pdns/speedtest.cc +++ b/pdns/speedtest.cc @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include "credentials.hh" #include "dnsparser.hh" #include "sstuff.hh" #include "misc.hh" @@ -11,6 +12,12 @@ #include "uuid-utils.hh" #include "dnssecinfra.hh" #include "lock.hh" +#include "dns_random.hh" +#include "arguments.hh" + +#if defined(HAVE_LIBSODIUM) +#include +#endif #ifndef RECURSOR #include "statbag.hh" @@ -35,8 +42,8 @@ template void doRun(const C& cmd, int mseconds=100) it.it_interval.tv_sec=0; it.it_interval.tv_usec=0; - signal(SIGVTALRM, alarmHandler); - setitimer(ITIMER_VIRTUAL, &it, 0); + signal(SIGPROF, alarmHandler); + setitimer(ITIMER_PROF, &it, 0); unsigned int runs=0; g_stop=false; @@ -1028,6 +1035,110 @@ struct ReadWriteLockSharedTryTest bool d_contended; }; +struct CredentialsHashTest +{ + explicit CredentialsHashTest() {} + + string getName() const + { + return "Credentials hashing test"; + } + + void operator()() const + { + hashPassword(d_password); + } + +private: + const std::string d_password{"test password"}; +}; + +struct RndSpeedTest +{ + explicit RndSpeedTest(std::string which) : name(which){ + ::arg().set("entropy-source", "If set, read entropy from this file")="/dev/urandom"; + ::arg().set("rng", "") = which; + dns_random_init("", true); + } + string getName() const + { + return "Random test " + name; + } + + void operator()() const + { + dns_random_uint16(); + } + + const std::string name; +}; + +struct CredentialsVerifyTest +{ + explicit CredentialsVerifyTest() { + d_hashed = hashPassword(d_password); + } + + string getName() const + { + return "Credentials verification test"; + } + + void operator()() const + { + verifyPassword(d_hashed, d_password); + } + +private: + std::string d_hashed; + const std::string d_password{"test password"}; +}; + +struct BurtleHashTest +{ + explicit BurtleHashTest(const string& str) : d_name(str) {} + + string getName() const + { + return "BurtleHash"; + } + + void operator()() const + { + burtle(reinterpret_cast(d_name.data()), d_name.length(), 0); + + } + +private: + const string d_name; +}; + + +#if defined(HAVE_LIBSODIUM) +struct SipHashTest +{ + explicit SipHashTest(const string& str) : d_name(str) + { + crypto_shorthash_keygen(d_key); + } + + string getName() const + { + return "SipHash"; + } + + void operator()() const + { + unsigned char out[crypto_shorthash_BYTES]; + crypto_shorthash(out, reinterpret_cast(d_name.data()), d_name.length(), d_key); + } + +private: + const string d_name; + unsigned char d_key[crypto_shorthash_KEYBYTES]; +}; +#endif + int main(int argc, char** argv) try { @@ -1134,6 +1245,20 @@ try doRun(UUIDGenTest()); +#if defined(HAVE_GETRANDOM) + doRun(RndSpeedTest("getrandom")); +#endif +#if defined(HAVE_ARC4RANDOM) + doRun(RndSpeedTest("arc4random")); +#endif +#if defined(HAVE_RANDOMBYTES_STIR) + doRun(RndSpeedTest("sodium")); +#endif +#if defined(HAVE_RAND_BYTES) + doRun(RndSpeedTest("openssl")); +#endif + doRun(RndSpeedTest("urandom")); + doRun(NSEC3HashTest(1, "ABCD")); doRun(NSEC3HashTest(10, "ABCD")); doRun(NSEC3HashTest(50, "ABCD")); @@ -1146,6 +1271,11 @@ try doRun(NSEC3HashTest(150, "ABCDABCDABCDABCDABCDABCDABCDABCD")); doRun(NSEC3HashTest(500, "ABCDABCDABCDABCDABCDABCDABCDABCD")); +#if defined(HAVE_LIBSODIUM) && defined(HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT) + doRun(CredentialsHashTest()); + doRun(CredentialsVerifyTest()); +#endif + #ifndef RECURSOR S.doRings(); @@ -1156,10 +1286,20 @@ try doRun(StatRingDNSNameQTypeTest(DNSName("example.com"), QType(1))); #endif - cerr<<"Total runs: " << g_totalRuns< SSQLite3::prepare(const string& query, int nparams __attribute__((unused))) { - return std::unique_ptr(new SSQLite3Statement(this, m_dolog, query)); + return std::make_unique(this, m_dolog, query); } void SSQLite3::execute(const string& query) { diff --git a/pdns/sstuff.hh b/pdns/sstuff.hh index f84d50c36409..5b14e234e60c 100644 --- a/pdns/sstuff.hh +++ b/pdns/sstuff.hh @@ -65,6 +65,9 @@ public: Socket& operator=(Socket&& rhs) { + if (d_socket != -1) { + close(d_socket); + } d_socket = rhs.d_socket; rhs.d_socket = -1; d_buffer = std::move(rhs.d_buffer); @@ -96,7 +99,7 @@ public: throw NetworkError("Accepting a connection: "+stringerror()); } - return std::unique_ptr(new Socket(s)); + return std::make_unique(s); } //! Get remote address diff --git a/pdns/stat_t.hh b/pdns/stat_t.hh index 2a765c26a6df..d318c980b259 100644 --- a/pdns/stat_t.hh +++ b/pdns/stat_t.hh @@ -21,6 +21,8 @@ */ #pragma once +#include + #define CPU_LEVEL1_DCACHE_LINESIZE 64 // Until we know better via configure/getconf namespace pdns { diff --git a/pdns/statbag.cc b/pdns/statbag.cc index 28f46dfefc6e..46066d2ca076 100644 --- a/pdns/statbag.cc +++ b/pdns/statbag.cc @@ -44,10 +44,9 @@ StatBag::StatBag() void StatBag::exists(const string &key) { - if(!d_keyDescriptions.count(key)) - { - throw PDNSException("Trying to deposit into unknown StatBag key '"+key+"'"); - } + if (!d_keyDescriptions.count(key)) { + throw PDNSException("Trying to deposit into unknown StatBag key '"+key+"'"); + } } string StatBag::directory(const string &prefix) @@ -148,20 +147,12 @@ unsigned long StatBag::read(const string &key) { exists(key); funcstats_t::const_iterator iter = d_funcstats.find(key); - if(iter != d_funcstats.end()) + if (iter != d_funcstats.end()) { return iter->second(iter->first); + } return *d_stats[key]; } -unsigned long StatBag::readZero(const string &key) -{ - exists(key); - unsigned long tmp=*d_stats[key]; - d_stats[key]=nullptr; - return tmp; -} - - string StatBag::getValueStr(const string &key) { ostringstream o; @@ -169,13 +160,6 @@ string StatBag::getValueStr(const string &key) return o.str(); } -string StatBag::getValueStrZero(const string &key) -{ - ostringstream o; - o<::StatRing(unsigned int size) d_items.set_capacity(size); } -template -StatRing::StatRing(const StatRing &arg) -{ - std::lock_guard thislock(d_lock); - std::lock_guard arglock(arg.d_lock); - - d_items = arg.d_items; - d_help = arg.d_help; -} - template void StatRing::account(const T& t) { - std::lock_guard l(d_lock); d_items.push_back(t); } template uint64_t StatRing::getSize() const { - std::lock_guard l(d_lock); return d_items.capacity(); } template uint64_t StatRing::getEntriesCount() const { - std::lock_guard l(d_lock); return d_items.size(); } template void StatRing::resize(unsigned int newsize) { - std::lock_guard l(d_lock); d_items.set_capacity(newsize); } template void StatRing::setHelp(const string &str) { - d_help=str; + d_help = str; } template -string StatRing::getHelp() +string StatRing::getHelp() const { return d_help; } @@ -246,9 +216,8 @@ string StatRing::getHelp() template vector >StatRing::get() const { - std::lock_guard l(d_lock); map res; - for(typename boost::circular_buffer::const_iterator i=d_items.begin();i!=d_items.end();++i) { + for (typename boost::circular_buffer::const_iterator i = d_items.begin(); i != d_items.end(); ++i) { res[*i]++; } @@ -270,41 +239,41 @@ void StatBag::registerRingStats(const string& name) void StatBag::declareRing(const string &name, const string &help, unsigned int size) { d_rings.emplace(name, size); - d_rings[name].setHelp(help); + d_rings[name].lock()->setHelp(help); registerRingStats(name); } void StatBag::declareComboRing(const string &name, const string &help, unsigned int size) { d_comboRings.emplace(name, size); - d_comboRings[name].setHelp(help); + d_comboRings[name].lock()->setHelp(help); registerRingStats(name); } void StatBag::declareDNSNameQTypeRing(const string &name, const string &help, unsigned int size) { d_dnsnameqtyperings.emplace(name, size); - d_dnsnameqtyperings[name].setHelp(help); + d_dnsnameqtyperings[name].lock()->setHelp(help); registerRingStats(name); } vector > StatBag::getRing(const string &name) { - if(d_rings.count(name)) { - return d_rings[name].get(); + if (d_rings.count(name)) { + return d_rings[name].lock()->get(); } vector > ret; if (d_comboRings.count(name)) { typedef pair stor_t; - vector raw =d_comboRings[name].get(); + vector raw =d_comboRings[name].lock()->get(); for(const stor_t& stor : raw) { - ret.push_back(make_pair(stor.first.ca.toString(), stor.second)); + ret.emplace_back(stor.first.ca.toString(), stor.second); } - } else if(d_dnsnameqtyperings.count(name)) { - auto raw = d_dnsnameqtyperings[name].get(); + } else if (d_dnsnameqtyperings.count(name)) { + auto raw = d_dnsnameqtyperings[name].lock()->get(); for (auto const &e : raw) { - ret.push_back(make_pair(std::get<0>(e.first).toLogString() + "/" + std::get<1>(e.first).toString(), e.second)); + ret.emplace_back(std::get<0>(e.first).toLogString() + "/" + std::get<1>(e.first).toString(), e.second); } } return ret; @@ -313,65 +282,64 @@ vector > StatBag::getRing(const string &name) template void StatRing::reset() { - std::lock_guard l(d_lock); d_items.clear(); } void StatBag::resetRing(const string &name) { if(d_rings.count(name)) - d_rings[name].reset(); + d_rings[name].lock()->reset(); if(d_comboRings.count(name)) - d_comboRings[name].reset(); + d_comboRings[name].lock()->reset(); if(d_dnsnameqtyperings.count(name)) - d_dnsnameqtyperings[name].reset(); + d_dnsnameqtyperings[name].lock()->reset(); } void StatBag::resizeRing(const string &name, unsigned int newsize) { if(d_rings.count(name)) - d_rings[name].resize(newsize); + d_rings[name].lock()->resize(newsize); if(d_comboRings.count(name)) - d_comboRings[name].resize(newsize); + d_comboRings[name].lock()->resize(newsize); if(d_dnsnameqtyperings.count(name)) - return d_dnsnameqtyperings[name].resize(newsize); + return d_dnsnameqtyperings[name].lock()->resize(newsize); } uint64_t StatBag::getRingSize(const string &name) { if(d_rings.count(name)) - return d_rings[name].getSize(); + return d_rings[name].lock()->getSize(); if(d_comboRings.count(name)) - return d_comboRings[name].getSize(); + return d_comboRings[name].lock()->getSize(); if(d_dnsnameqtyperings.count(name)) - return d_dnsnameqtyperings[name].getSize(); + return d_dnsnameqtyperings[name].lock()->getSize(); return 0; } uint64_t StatBag::getRingEntriesCount(const string &name) { if(d_rings.count(name)) - return d_rings[name].getEntriesCount(); + return d_rings[name].lock()->getEntriesCount(); if(d_comboRings.count(name)) - return d_comboRings[name].getEntriesCount(); + return d_comboRings[name].lock()->getEntriesCount(); if(d_dnsnameqtyperings.count(name)) - return d_dnsnameqtyperings[name].getEntriesCount(); + return d_dnsnameqtyperings[name].lock()->getEntriesCount(); return 0; } string StatBag::getRingTitle(const string &name) { if(d_rings.count(name)) - return d_rings[name].getHelp(); + return d_rings[name].lock()->getHelp(); if(d_comboRings.count(name)) - return d_comboRings[name].getHelp(); + return d_comboRings[name].lock()->getHelp(); if(d_dnsnameqtyperings.count(name)) - return d_dnsnameqtyperings[name].getHelp(); + return d_dnsnameqtyperings[name].lock()->getHelp(); return ""; } -vectorStatBag::listRings() +vectorStatBag::listRings() const { vector ret; for(auto & d_ring : d_rings) @@ -384,7 +352,7 @@ vectorStatBag::listRings() return ret; } -bool StatBag::ringExists(const string &name) +bool StatBag::ringExists(const string &name) const { return d_rings.count(name) || d_comboRings.count(name) || d_dnsnameqtyperings.count(name); } diff --git a/pdns/statbag.hh b/pdns/statbag.hh index ce5030dbaab2..acdc3832303b 100644 --- a/pdns/statbag.hh +++ b/pdns/statbag.hh @@ -20,9 +20,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #pragma once -#include #include -#include #include #include #include @@ -31,15 +29,15 @@ #include "iputils.hh" #include "circular_buffer.hh" - template > class StatRing { public: StatRing(unsigned int size=10000); - // Some older C++ libs have trouble emplacing without a copy-constructor, so provide one - StatRing(const StatRing &); - StatRing & operator=(const StatRing &) = delete; + StatRing(const StatRing&) = delete; + StatRing& operator=(const StatRing&) = delete; + StatRing& operator=(StatRing&&) = delete; + StatRing(StatRing&&) = default; void account(const T &item); @@ -48,7 +46,7 @@ public: void resize(unsigned int newsize); void reset(); void setHelp(const string &str); - string getHelp(); + string getHelp() const; vector > get() const; private: @@ -58,7 +56,6 @@ private: } boost::circular_buffer d_items; - mutable std::mutex d_lock; string d_help; }; @@ -73,9 +70,9 @@ class StatBag map> d_stats; map d_keyDescriptions; map d_statTypes; - map >d_rings; - map >d_comboRings; - map > >d_dnsnameqtyperings; + map > > d_rings; + map > > d_comboRings; + map > > > d_dnsnameqtyperings; typedef boost::function func_t; typedef map funcstats_t; funcstats_t d_funcstats; @@ -98,27 +95,33 @@ public: string getRingTitle(const string &name); void ringAccount(const char* name, const string &item) { - if(d_doRings) { - if(!d_rings.count(name)) + if (d_doRings) { + auto it = d_rings.find(name); + if (it == d_rings.end()) { throw runtime_error("Attempting to account to non-existent ring '"+std::string(name)+"'"); + } - d_rings[name].account(item); + it->second.lock()->account(item); } } void ringAccount(const char* name, const ComboAddress &item) { - if(d_doRings) { - if(!d_comboRings.count(name)) + if (d_doRings) { + auto it = d_comboRings.find(name); + if (it == d_comboRings.end()) { throw runtime_error("Attempting to account to non-existent comboRing '"+std::string(name)+"'"); - d_comboRings[name].account(item); + } + it->second.lock()->account(item); } } void ringAccount(const char* name, const DNSName &dnsname, const QType &qtype) { - if(d_doRings) { - if(!d_dnsnameqtyperings.count(name)) + if (d_doRings) { + auto it = d_dnsnameqtyperings.find(name); + if (it == d_dnsnameqtyperings.end()) { throw runtime_error("Attempting to account to non-existent dnsname+qtype ring '"+std::string(name)+"'"); - d_dnsnameqtyperings[name].account(std::make_tuple(dnsname, qtype)); + } + it->second.lock()->account(std::make_tuple(dnsname, qtype)); } } @@ -127,8 +130,8 @@ public: d_doRings=true; } - vectorlistRings(); - bool ringExists(const string &name); + vectorlistRings() const; + bool ringExists(const string &name) const; void resetRing(const string &name); void resizeRing(const string &name, unsigned int newsize); uint64_t getRingSize(const string &name); @@ -143,10 +146,8 @@ public: inline void inc(const string &key); //!< increase this key's value by one void set(const string &key, unsigned long value); //!< set this key's value unsigned long read(const string &key); //!< read the value behind this key - unsigned long readZero(const string &key); //!< read the value behind this key, and zero it afterwards AtomicCounter *getPointer(const string &key); //!< get a direct pointer to the value behind a key. Use this for high performance increments string getValueStr(const string &key); //!< read a value behind a key, and return it as a string - string getValueStrZero(const string &key); //!< read a value behind a key, and return it as a string, and zero afterwards void blacklist(const string &str); bool d_allowRedeclare; // only set this true during tests, never in production code @@ -161,5 +162,5 @@ inline void StatBag::deposit(const string &key, int value) inline void StatBag::inc(const string &key) { - deposit(key,1); + deposit(key, 1); } diff --git a/pdns/stubresolver.cc b/pdns/stubresolver.cc index 4b3a47ab7ad8..af5f2ea09632 100644 --- a/pdns/stubresolver.cc +++ b/pdns/stubresolver.cc @@ -22,8 +22,7 @@ // s_resolversForStub contains the ComboAddresses that are used by // stubDoResolve -static vector s_resolversForStub; -static ReadWriteLock s_resolversForStubLock; +static SharedLockGuarded> s_resolversForStub; static bool s_stubResolvConfigured = false; // /etc/resolv.conf last modification time @@ -37,8 +36,7 @@ static string logPrefix = "[stub-resolver] "; */ bool resolversDefined() { - ReadLock l(&s_resolversForStubLock); - if (s_resolversForStub.empty()) { + if (s_resolversForStub.read_lock()->empty()) { g_log<& resolversForStub, const time_t& now) { struct stat st; s_localResolvConfLastCheck = now; @@ -63,7 +61,7 @@ static void parseLocalResolvConf_locked(const time_t& now) return; } - s_resolversForStub = std::move(resolvers); + resolversForStub = std::move(resolvers); } } } @@ -74,8 +72,7 @@ static void parseLocalResolvConf() if ((s_localResolvConfLastCheck + LOCAL_RESOLV_CONF_MAX_CHECK_INTERVAL) > now) return ; - WriteLock wl(&s_resolversForStubLock); - parseLocalResolvConf_locked(now); + parseLocalResolvConf_locked(*(s_resolversForStub.write_lock()), now); } @@ -90,14 +87,14 @@ static void parseLocalResolvConf() void stubParseResolveConf() { if(::arg().mustDo("resolver")) { - WriteLock wl(&s_resolversForStubLock); + auto resolversForStub = s_resolversForStub.write_lock(); vector parts; stringtok(parts, ::arg()["resolver"], " ,\t"); for (const auto& addr : parts) - s_resolversForStub.push_back(ComboAddress(addr, 53)); + resolversForStub->push_back(ComboAddress(addr, 53)); } - if (s_resolversForStub.empty()) { + if (s_resolversForStub.read_lock()->empty()) { parseLocalResolvConf(); } // Emit a warning if there are no stubs. @@ -119,7 +116,7 @@ int stubDoResolve(const DNSName& qname, uint16_t qtype, vector& r if (!resolversDefined()) return RCode::ServFail; - ReadLock l(&s_resolversForStubLock); + auto resolversForStub = s_resolversForStub.read_lock(); vector packet; DNSPacketWriter pw(packet, qname, qtype); @@ -128,12 +125,12 @@ int stubDoResolve(const DNSName& qname, uint16_t qtype, vector& r string queryNameType = qname.toString() + "|" + QType(qtype).toString(); string msg ="Doing stub resolving for '" + queryNameType + "', using resolvers: "; - for (const auto& server : s_resolversForStub) { + for (const auto& server : *resolversForStub) { msg += server.toString() + ", "; } g_log<& r string reply; - waitForData(sock.getHandle(), 2, 0); + // error handled after this + (void)waitForData(sock.getHandle(), 2, 0); try { retry: sock.read(reply); // this calls recv diff --git a/pdns/svc-records.cc b/pdns/svc-records.cc index 90d1e77a9829..0f610dead547 100644 --- a/pdns/svc-records.cc +++ b/pdns/svc-records.cc @@ -34,12 +34,19 @@ const std::map SvcParam::SvcParams = { }; SvcParam::SvcParamKey SvcParam::keyFromString(const std::string& k) { + bool ignored; + return SvcParam::keyFromString(k, ignored); +} + +SvcParam::SvcParamKey SvcParam::keyFromString(const std::string& k, bool &generic) { auto it = SvcParams.find(k); if (it != SvcParams.end()) { + generic = false; return it->second; } if (k.substr(0, 3) == "key") { try { + generic = true; return SvcParam::SvcParamKey(pdns_stou(k.substr(3))); } catch (...) { @@ -56,8 +63,6 @@ std::string SvcParam::keyToString(const SvcParam::SvcParamKey& k) { return "key" + std::to_string(k); } -SvcParam::SvcParam() {}; - SvcParam::SvcParam(const SvcParamKey &key) { d_key = key; if (d_key != SvcParamKey::no_default_alpn) { @@ -104,7 +109,7 @@ SvcParam::SvcParam(const SvcParamKey &key, std::set &&value) { SvcParam::SvcParam(const SvcParamKey &key, std::set &&value) { d_key = key; if (d_key != SvcParamKey::mandatory) { - throw std::invalid_argument("can not create SvcParam for " + keyToString(key) + " with a string-set value"); + throw std::invalid_argument("can not create SvcParam for " + keyToString(key) + " with a SvcParamKey-set value"); } d_mandatory = std::move(value); } diff --git a/pdns/svc-records.hh b/pdns/svc-records.hh index 5f154c99f8fb..19de2b9869e6 100644 --- a/pdns/svc-records.hh +++ b/pdns/svc-records.hh @@ -42,7 +42,7 @@ class SvcParam { }; //! empty Param, unusable - SvcParam(); + SvcParam() = delete; //! To create a value-less SvcParam (like no-default-alpn) SvcParam(const SvcParamKey &key); @@ -68,6 +68,9 @@ class SvcParam { //! Returns the SvcParamKey based on the input static SvcParamKey keyFromString(const std::string &k); + //! Returns the SvcParamKey based on the input, generic is true when the format was 'keyNNNN' + static SvcParamKey keyFromString(const std::string &k, bool &generic); + //! Returns the string value of the SvcParamKey static std::string keyToString(const SvcParamKey &k); diff --git a/pdns/syncres.cc b/pdns/syncres.cc index 1f3e44455651..5f7a14f40723 100644 --- a/pdns/syncres.cc +++ b/pdns/syncres.cc @@ -65,22 +65,22 @@ unsigned int SyncRes::s_serverdownthrottletime; unsigned int SyncRes::s_nonresolvingnsmaxfails; unsigned int SyncRes::s_nonresolvingnsthrottletime; unsigned int SyncRes::s_ecscachelimitttl; -std::atomic SyncRes::s_authzonequeries; -std::atomic SyncRes::s_queries; -std::atomic SyncRes::s_outgoingtimeouts; -std::atomic SyncRes::s_outgoing4timeouts; -std::atomic SyncRes::s_outgoing6timeouts; -std::atomic SyncRes::s_outqueries; -std::atomic SyncRes::s_tcpoutqueries; -std::atomic SyncRes::s_dotoutqueries; -std::atomic SyncRes::s_throttledqueries; -std::atomic SyncRes::s_dontqueries; -std::atomic SyncRes::s_qnameminfallbacksuccess; -std::atomic SyncRes::s_unreachables; -std::atomic SyncRes::s_ecsqueries; -std::atomic SyncRes::s_ecsresponses; -std::map> SyncRes::s_ecsResponsesBySubnetSize4; -std::map> SyncRes::s_ecsResponsesBySubnetSize6; +pdns::stat_t SyncRes::s_authzonequeries; +pdns::stat_t SyncRes::s_queries; +pdns::stat_t SyncRes::s_outgoingtimeouts; +pdns::stat_t SyncRes::s_outgoing4timeouts; +pdns::stat_t SyncRes::s_outgoing6timeouts; +pdns::stat_t SyncRes::s_outqueries; +pdns::stat_t SyncRes::s_tcpoutqueries; +pdns::stat_t SyncRes::s_dotoutqueries; +pdns::stat_t SyncRes::s_throttledqueries; +pdns::stat_t SyncRes::s_dontqueries; +pdns::stat_t SyncRes::s_qnameminfallbacksuccess; +pdns::stat_t SyncRes::s_unreachables; +pdns::stat_t SyncRes::s_ecsqueries; +pdns::stat_t SyncRes::s_ecsresponses; +std::map SyncRes::s_ecsResponsesBySubnetSize4; +std::map SyncRes::s_ecsResponsesBySubnetSize6; uint8_t SyncRes::s_ecsipv4limit; uint8_t SyncRes::s_ecsipv6limit; @@ -100,6 +100,7 @@ unsigned int SyncRes::s_refresh_ttlperc; int SyncRes::s_tcp_fast_open; bool SyncRes::s_tcp_fast_open_connect; bool SyncRes::s_dot_to_port_853; +int SyncRes::s_event_trace_enabled; #define LOG(x) if(d_lm == Log) { g_log <&ret, unsigned int depth) { + d_eventTrace.add(RecEventTrace::SyncRes); vState state = vState::Indeterminate; s_queries++; d_wasVariable=false; @@ -170,6 +172,7 @@ int SyncRes::beginResolve(const DNSName &qname, const QType qtype, QClass qclass } } + d_eventTrace.add(RecEventTrace::SyncRes, res, false); return res; } @@ -201,16 +204,16 @@ bool SyncRes::doSpecialNamesResolve(const DNSName &qname, const QType qtype, con qclass == QClass::IN) { handled = true; if (qtype == QType::PTR || qtype == QType::ANY) - answers.push_back({QType::PTR, "localhost."}); + answers.emplace_back(QType::PTR, "localhost."); } if (qname.isPartOf(localhost) && qclass == QClass::IN) { handled = true; if (qtype == QType::A || qtype == QType::ANY) - answers.push_back({QType::A, "127.0.0.1"}); + answers.emplace_back(QType::A, "127.0.0.1"); if (qtype == QType::AAAA || qtype == QType::ANY) - answers.push_back({QType::AAAA, "::1"}); + answers.emplace_back(QType::AAAA, "::1"); } if ((qname == versionbind || qname == idserver || qname == versionpdns) && @@ -218,9 +221,9 @@ bool SyncRes::doSpecialNamesResolve(const DNSName &qname, const QType qtype, con handled = true; if (qtype == QType::TXT || qtype == QType::ANY) { if(qname == versionbind || qname == versionpdns) - answers.push_back({QType::TXT, "\""+::arg()["version-string"]+"\""}); + answers.emplace_back(QType::TXT, "\"" + ::arg()["version-string"] + "\""); else if (s_serverID != "disabled") - answers.push_back({QType::TXT, "\""+s_serverID+"\""}); + answers.emplace_back(QType::TXT, "\"" + s_serverID + "\""); } } @@ -238,7 +241,7 @@ bool SyncRes::doSpecialNamesResolve(const DNSName &qname, const QType qtype, con ans< SyncRes::getAddrs(const DNSName &qname, unsigned int depth, try { // First look for both A and AAAA in the cache and be satisfied if we find anything res_t cset; - if (s_doIPv4 && g_recCache->get(d_now.tv_sec, qname, QType::A, false, &cset, d_cacheRemote, d_refresh, d_routingTag) > 0) { + if (s_doIPv4 && g_recCache->get(d_now.tv_sec, qname, QType::A, false, &cset, d_cacheRemote, false, d_routingTag) > 0) { for (const auto &i : cset) { if (auto rec = getRR(i)) { ret.push_back(rec->getCA(53)); } } } - if (s_doIPv6 && g_recCache->get(d_now.tv_sec, qname, QType::AAAA, false, &cset, d_cacheRemote, d_refresh, d_routingTag) > 0) { + if (s_doIPv6 && g_recCache->get(d_now.tv_sec, qname, QType::AAAA, false, &cset, d_cacheRemote, false, d_routingTag) > 0) { for (const auto &i : cset) { if (auto rec = getRR(i)) { ret.push_back(rec->getCA(53)); @@ -1136,7 +1139,7 @@ vector SyncRes::getAddrs(const DNSName &qname, unsigned int depth, // We have some IPv4 records, don't bother with going out to get IPv6, but do consult the cache, we might have // encountered some IPv6 glue cset.clear(); - if (g_recCache->get(d_now.tv_sec, qname, QType::AAAA, false, &cset, d_cacheRemote, d_refresh, d_routingTag) > 0) { + if (g_recCache->get(d_now.tv_sec, qname, QType::AAAA, false, &cset, d_cacheRemote, false, d_routingTag) > 0) { for (const auto &i : cset) { if (auto rec = getRR(i)) { ret.push_back(rec->getCA(53)); @@ -1219,7 +1222,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType qtype, vector vector ns; *flawedNSSet = false; - if(g_recCache->get(d_now.tv_sec, subdomain, QType::NS, false, &ns, d_cacheRemote, d_refresh, d_routingTag) > 0) { + if(g_recCache->get(d_now.tv_sec, subdomain, QType::NS, false, &ns, d_cacheRemote, false, d_routingTag) > 0) { bestns.reserve(ns.size()); for(auto k=ns.cbegin();k!=ns.cend(); ++k) { @@ -1235,7 +1238,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType qtype, vector const DNSRecord& dr=*k; auto nrr = getRR(dr); if(nrr && (!nrr->getNS().isPartOf(subdomain) || g_recCache->get(d_now.tv_sec, nrr->getNS(), nsqt, - false, doLog() ? &aset : 0, d_cacheRemote, d_refresh, d_routingTag) > 5)) { + false, doLog() ? &aset : 0, d_cacheRemote, false, d_routingTag) > 5)) { bestns.push_back(dr); LOG(prefix< '"<getNS()<<"'"<getNS().isPartOf(subdomain)); @@ -1259,7 +1262,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType qtype, vector answer.qtype=qtype.getCode(); for(const auto& dr : bestns) { if (auto nsContent = getRR(dr)) { - answer.bestns.insert(make_pair(dr.d_name, nsContent->getNS())); + answer.bestns.emplace(dr.d_name, nsContent->getNS()); } } @@ -1289,7 +1292,7 @@ void SyncRes::getBestNSFromCache(const DNSName &qname, const QType qtype, vector LOG(prefix<> SyncRes::shuffleInSpeedOrder(NsSet rnameservers.reserve(tnameservers.size()); for(const auto& tns: tnameservers) { float speed = t_sstorage.nsSpeeds[tns.first].get(d_now); - rnameservers.push_back({tns.first, speed}); + rnameservers.emplace_back(tns.first, speed); if(tns.first.empty()) // this was an authoritative OOB zone, don't pollute the nsSpeeds with that return rnameservers; } @@ -2582,10 +2585,10 @@ vState SyncRes::getDSRecords(const DNSName& zone, dsmap_t& ds, bool taOnly, unsi /* RFC 4509 section 3: "Validator implementations SHOULD ignore DS RRs containing SHA-1 * digests if DS RRs with SHA-256 digests are present in the DS RRset." - * As SHA348 is specified as well, the spirit of the this line is "use the best algorithm". + * We interpret that as: do not use SHA-1 if SHA-256 or SHA-384 is available */ for (auto dsrec = ds.begin(); dsrec != ds.end(); ) { - if (dsrec->d_digesttype != bestDigestType) { + if (dsrec->d_digesttype == DNSSECKeeper::DIGEST_SHA1 && dsrec->d_digesttype != bestDigestType) { dsrec = ds.erase(dsrec); } else { @@ -2857,12 +2860,18 @@ vState SyncRes::validateRecordsWithSigs(unsigned int depth, const DNSName& qname state = vState::BogusSelfSignedDS; dsFailed = true; } - else if (qtype == QType::DS && signer == qname && !signer.isRoot() && (type == QType::SOA || type == QType::NSEC || type == QType::NSEC3)) { + else if (qtype == QType::DS && signer == qname && !signer.isRoot()) { + if (type == QType::SOA || type == QType::NSEC || type == QType::NSEC3) { /* if we are trying to validate the DS or more likely NSEC(3)s proving that it does not exist, we have a problem. In that case let's go Bogus (we will check later if we missed a cut) */ - state = vState::BogusSelfSignedDS; - dsFailed = true; + state = vState::BogusSelfSignedDS; + dsFailed = true; + } + else if (type == QType::CNAME) { + state = vState::BogusUnableToGetDSs; + dsFailed = true; + } } else if (qtype == QType::DNSKEY && signer == qname) { /* that actually does happen when a server returns NS records in authority @@ -3363,16 +3372,21 @@ RCode::rcodes_ SyncRes::updateCacheFromRecords(unsigned int depth, LWResult& lwr isAA = false; expectSignature = false; } + else if (isDNAMEAnswer && (i->first.place != DNSResourceRecord::ANSWER || i->first.type != QType::DNAME || !qname.isPartOf(i->first.name))) { + /* see above */ + isAA = false; + expectSignature = false; + } - if (isCNAMEAnswer && i->first.place == DNSResourceRecord::AUTHORITY && i->first.type == QType::NS && auth == i->first.name) { - /* These NS can't be authoritative since we have a CNAME answer for which (see above) only the + if ((isCNAMEAnswer || isDNAMEAnswer) && i->first.place == DNSResourceRecord::AUTHORITY && i->first.type == QType::NS && auth == i->first.name) { + /* These NS can't be authoritative since we have a CNAME/DNAME answer for which (see above) only the record describing that alias is necessarily authoritative. But if we allow the current auth, which might be serving the child zone, to raise the TTL of non-authoritative NS in the cache, they might be able to keep a "ghost" zone alive forever, even after the delegation is gone from the parent. So let's just do nothing with them, we can fetch them directly if we need them. */ - LOG(d_prefix<<": skipping authority NS from '"<first.name<<"|"<first.type)<first.name<<"|"<first.type)<add(ne); if (s_rootNXTrust && ne.d_auth.isRoot() && auth.isRoot() && lwr.d_aabit) { ne.d_name = ne.d_name.getLastLabel(); @@ -3803,9 +3817,7 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co } LOG(prefix<add(ne); - } + g_negCache->add(ne); /* Careful! If the client is asking for a DS that does not exist, we need to provide the SOA along with the NSEC(3) proof and we might not have it if we picked up the proof from a delegation, in which case we need to keep on to do the actual DS @@ -3857,10 +3869,8 @@ bool SyncRes::processRecords(const std::string& prefix, const DNSName& qname, co } ne.d_ttd = d_now.tv_sec + lowestTTL; - if (!wasVariable()) { - if (qtype.getCode()) { // prevents us from NXDOMAIN'ing a whole domain - g_negCache->add(ne); - } + if (qtype.getCode()) { // prevents us from NXDOMAIN'ing a whole domain + g_negCache->add(ne); } ret.push_back(rec); @@ -3927,7 +3937,7 @@ bool SyncRes::doResolveAtThisIP(const std::string& prefix, const DNSName& qname, } int preOutQueryRet = RCode::NoError; - if(d_pdl && d_pdl->preoutquery(remoteIP, d_requestor, qname, qtype, doTCP, lwr.d_records, preOutQueryRet)) { + if(d_pdl && d_pdl->preoutquery(remoteIP, d_requestor, qname, qtype, doTCP, lwr.d_records, preOutQueryRet, d_eventTrace)) { LOG(prefix< ret; - int res=-1; + int res = -1; try { - res=sr.beginResolve(g_rootdnsname, QType::NS, 1, ret, depth + 1); + res = sr.beginResolve(g_rootdnsname, QType::NS, 1, ret, depth + 1); if (g_dnssecmode != DNSSECMode::Off && g_dnssecmode != DNSSECMode::ProcessNoValidate) { auto state = sr.getValidationState(); if (vStateIsBogus(state)) { @@ -4637,11 +4647,11 @@ int SyncRes::getRootNS(struct timeval now, asyncresolve_t asyncCallback, unsigne g_log<(new NetmaskGroup()); + s_dontQuery = std::make_unique(); s_dontQuery->addMask(mask); } static void addDontQuery(const Netmask& mask) { if (!s_dontQuery) - s_dontQuery = std::unique_ptr(new NetmaskGroup()); + s_dontQuery = std::make_unique(); s_dontQuery->addMask(mask); } @@ -734,22 +736,22 @@ public: static thread_local ThreadLocalStorage t_sstorage; - static std::atomic s_queries; - static std::atomic s_outgoingtimeouts; - static std::atomic s_outgoing4timeouts; - static std::atomic s_outgoing6timeouts; - static std::atomic s_throttledqueries; - static std::atomic s_dontqueries; - static std::atomic s_qnameminfallbacksuccess; - static std::atomic s_authzonequeries; - static std::atomic s_outqueries; - static std::atomic s_tcpoutqueries; - static std::atomic s_dotoutqueries; - static std::atomic s_unreachables; - static std::atomic s_ecsqueries; - static std::atomic s_ecsresponses; - static std::map> s_ecsResponsesBySubnetSize4; - static std::map> s_ecsResponsesBySubnetSize6; + static pdns::stat_t s_queries; + static pdns::stat_t s_outgoingtimeouts; + static pdns::stat_t s_outgoing4timeouts; + static pdns::stat_t s_outgoing6timeouts; + static pdns::stat_t s_throttledqueries; + static pdns::stat_t s_dontqueries; + static pdns::stat_t s_qnameminfallbacksuccess; + static pdns::stat_t s_authzonequeries; + static pdns::stat_t s_outqueries; + static pdns::stat_t s_tcpoutqueries; + static pdns::stat_t s_dotoutqueries; + static pdns::stat_t s_unreachables; + static pdns::stat_t s_ecsqueries; + static pdns::stat_t s_ecsresponses; + static std::map s_ecsResponsesBySubnetSize4; + static std::map s_ecsResponsesBySubnetSize6; static string s_serverID; static unsigned int s_minimumTTL; @@ -788,11 +790,16 @@ public: static int s_tcp_fast_open; static bool s_tcp_fast_open_connect; static bool s_dot_to_port_853; - + + static const int event_trace_to_pb = 1; + static const int event_trace_to_log = 2; + static int s_event_trace_enabled; + std::unordered_map d_discardedPolicies; DNSFilterEngine::Policy d_appliedPolicy; std::unordered_set d_policyTags; boost::optional d_routingTag; + RecEventTrace d_eventTrace; unsigned int d_authzonequeries; unsigned int d_outqueries; @@ -899,8 +906,8 @@ private: ostringstream d_trace; shared_ptr d_pdl; boost::optional d_outgoingECSNetwork; - std::shared_ptr>> d_outgoingProtobufServers{nullptr}; - std::shared_ptr>> d_frameStreamServers{nullptr}; + std::shared_ptr>> d_outgoingProtobufServers; + std::shared_ptr>> d_frameStreamServers; boost::optional d_initialRequestId; asyncresolve_t d_asyncResolve{nullptr}; struct timeval d_now; @@ -1023,9 +1030,9 @@ MT_t* getMT(); struct RecursorStats { - std::atomic servFails; - std::atomic nxDomains; - std::atomic noErrors; + pdns::stat_t servFails; + pdns::stat_t nxDomains; + pdns::stat_t noErrors; pdns::AtomicHistogram answers; pdns::AtomicHistogram auth4Answers; pdns::AtomicHistogram auth6Answers; @@ -1033,59 +1040,62 @@ struct RecursorStats pdns::AtomicHistogram cumulativeAnswers; pdns::AtomicHistogram cumulativeAuth4Answers; pdns::AtomicHistogram cumulativeAuth6Answers; - std::atomic avgLatencyUsec; - std::atomic avgLatencyOursUsec; - std::atomic qcounter; // not increased for unauth packets - std::atomic ipv6qcounter; - std::atomic tcpqcounter; - std::atomic unauthorizedUDP; // when this is increased, qcounter isn't - std::atomic unauthorizedTCP; // when this is increased, qcounter isn't - std::atomic policyDrops; - std::atomic tcpClientOverflow; - std::atomic clientParseError; - std::atomic serverParseError; - std::atomic tooOldDrops; - std::atomic truncatedDrops; - std::atomic queryPipeFullDrops; - std::atomic unexpectedCount; - std::atomic caseMismatchCount; - std::atomic spoofCount; - std::atomic resourceLimits; - std::atomic overCapacityDrops; - std::atomic ipv6queries; - std::atomic chainResends; - std::atomic nsSetInvalidations; - std::atomic ednsPingMatches; - std::atomic ednsPingMismatches; - std::atomic noPingOutQueries, noEdnsOutQueries; - std::atomic packetCacheHits; - std::atomic noPacketError; - std::atomic ignoredCount; - std::atomic emptyQueriesCount; + pdns::stat_t_trait avgLatencyUsec; + pdns::stat_t_trait avgLatencyOursUsec; + pdns::stat_t qcounter; // not increased for unauth packets + pdns::stat_t ipv6qcounter; + pdns::stat_t tcpqcounter; + pdns::stat_t unauthorizedUDP; // when this is increased, qcounter isn't + pdns::stat_t unauthorizedTCP; // when this is increased, qcounter isn't + pdns::stat_t sourceDisallowedNotify; // when this is increased, qcounter is also + pdns::stat_t zoneDisallowedNotify; // when this is increased, qcounter is also + pdns::stat_t policyDrops; + pdns::stat_t tcpClientOverflow; + pdns::stat_t clientParseError; + pdns::stat_t serverParseError; + pdns::stat_t tooOldDrops; + pdns::stat_t truncatedDrops; + pdns::stat_t queryPipeFullDrops; + pdns::stat_t unexpectedCount; + pdns::stat_t caseMismatchCount; + pdns::stat_t spoofCount; + pdns::stat_t resourceLimits; + pdns::stat_t overCapacityDrops; + pdns::stat_t ipv6queries; + pdns::stat_t chainResends; + pdns::stat_t nsSetInvalidations; + pdns::stat_t ednsPingMatches; + pdns::stat_t ednsPingMismatches; + pdns::stat_t noPingOutQueries, noEdnsOutQueries; + pdns::stat_t packetCacheHits; + pdns::stat_t noPacketError; + pdns::stat_t ignoredCount; + pdns::stat_t emptyQueriesCount; time_t startupTime; - std::atomic dnssecQueries; - std::atomic dnssecAuthenticDataQueries; - std::atomic dnssecCheckDisabledQueries; - std::atomic variableResponses; - std::atomic maxMThreadStackUsage; - std::atomic dnssecValidations; // should be the sum of all dnssecResult* stats - std::map > dnssecResults; - std::map > xdnssecResults; - std::map > policyResults; - LockGuarded>> policyHits; - std::atomic rebalancedQueries{0}; - std::atomic proxyProtocolInvalidCount{0}; - std::atomic nodLookupsDroppedOversize{0}; - std::atomic dns64prefixanswers{0}; + pdns::stat_t dnssecQueries; + pdns::stat_t dnssecAuthenticDataQueries; + pdns::stat_t dnssecCheckDisabledQueries; + pdns::stat_t variableResponses; + pdns::stat_t maxMThreadStackUsage; + pdns::stat_t dnssecValidations; // should be the sum of all dnssecResult* stats + std::map dnssecResults; + std::map xdnssecResults; + std::map policyResults; + LockGuarded> policyHits; + pdns::stat_t rebalancedQueries{0}; + pdns::stat_t proxyProtocolInvalidCount{0}; + pdns::stat_t nodLookupsDroppedOversize{0}; + pdns::stat_t dns64prefixanswers{0}; RecursorStats() : answers("answers", { 1000, 10000, 100000, 1000000 }), auth4Answers("auth4answers", { 1000, 10000, 100000, 1000000 }), auth6Answers("auth6answers", { 1000, 10000, 100000, 1000000 }), ourtime("ourtime", { 1000, 2000, 4000, 8000, 16000, 32000 }), - cumulativeAnswers("cumul-answers-", 10, 19), - cumulativeAuth4Answers("cumul-auth4answers-", 1000, 13), - cumulativeAuth6Answers("cumul-auth6answers-", 1000, 13) + cumulativeAnswers("cumul-clientanswers-", 10, 19), + // These two will be merged when outputting + cumulativeAuth4Answers("cumul-authanswers-", 1000, 13), + cumulativeAuth6Answers("cumul-authanswers-", 1000, 13) { } }; @@ -1119,7 +1129,7 @@ public: static unsigned int getCurrentConnections() { return s_currentConnections; } private: const int d_fd; - static AtomicCounter s_currentConnections; //!< total number of current TCP connections + static std::atomic s_currentConnections; //!< total number of current TCP connections }; class ImmediateServFailException @@ -1147,6 +1157,7 @@ extern thread_local std::unique_ptr t_servfailremotes, t_largeans extern thread_local std::unique_ptr > > t_queryring, t_servfailqueryring, t_bogusqueryring; extern thread_local std::shared_ptr t_allowFrom; +extern thread_local std::shared_ptr t_allowNotifyFrom; string doTraceRegex(vector::const_iterator begin, vector::const_iterator end); void parseACLs(); extern RecursorStats g_stats; @@ -1157,7 +1168,7 @@ extern std::atomic g_maxCacheEntries, g_maxPacketCacheEntries; extern bool g_lowercaseOutgoing; -std::string reloadAuthAndForwards(); +std::string reloadZoneConfiguration(); typedef boost::function pipefunc_t; void broadcastFunction(const pipefunc_t& func); void distributeAsyncFunction(const std::string& question, const pipefunc_t& func); @@ -1170,7 +1181,10 @@ int getFakePTRRecords(const DNSName& qname, vector& ret); template T broadcastAccFunction(const boost::function& func); -std::shared_ptr parseAuthAndForwards(); +typedef std::unordered_set notifyset_t; +std::tuple, std::shared_ptr> parseZoneConfiguration(); +void* pleaseSupplantAllowNotifyFor(std::shared_ptr ns); + uint64_t* pleaseGetNsSpeedsSize(); uint64_t* pleaseGetFailedServersSize(); uint64_t* pleaseGetEDNSStatusesSize(); @@ -1178,10 +1192,18 @@ uint64_t* pleaseGetConcurrentQueries(); uint64_t* pleaseGetThrottleSize(); uint64_t* pleaseGetPacketCacheHits(); uint64_t* pleaseGetPacketCacheSize(); -uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree, uint16_t qtype=0xffff); void doCarbonDump(void*); bool primeHints(time_t now = time(nullptr)); -void primeRootNSZones(bool, unsigned int depth); +void primeRootNSZones(DNSSECMode, unsigned int depth); + +struct WipeCacheResult +{ + int record_count = 0; + int negative_record_count = 0; + int packet_count = 0; +}; + +struct WipeCacheResult wipeCaches(const DNSName& canon, bool subtree, uint16_t qtype); extern __thread struct timeval g_now; diff --git a/pdns/tcpiohandler.cc b/pdns/tcpiohandler.cc index b5360fe9aa37..6068c1bc84ea 100644 --- a/pdns/tcpiohandler.cc +++ b/pdns/tcpiohandler.cc @@ -5,6 +5,8 @@ #include "lock.hh" #include "tcpiohandler.hh" +const bool TCPIOHandler::s_disableConnectForUnitTests = false; + #ifdef HAVE_LIBSODIUM #include #endif /* HAVE_LIBSODIUM */ @@ -20,6 +22,7 @@ #include "libssl.hh" + class OpenSSLFrontendContext { public: @@ -99,7 +102,7 @@ class OpenSSLTLSConnection: public TLSConnection } /* client-side connection */ - OpenSSLTLSConnection(const std::string& hostname, int socket, const struct timeval& timeout, SSL_CTX* tlsCtx): d_conn(std::unique_ptr(SSL_new(tlsCtx), SSL_free)), d_hostname(hostname), d_timeout(timeout) + OpenSSLTLSConnection(const std::string& hostname, int socket, const struct timeval& timeout, std::shared_ptr& tlsCtx): d_tlsCtx(tlsCtx), d_conn(std::unique_ptr(SSL_new(tlsCtx.get()), SSL_free)), d_hostname(hostname), d_timeout(timeout) { d_socket = socket; @@ -151,7 +154,12 @@ class OpenSSLTLSConnection: public TLSConnection return IOState::NeedWrite; } else if (error == SSL_ERROR_SYSCALL) { - throw std::runtime_error("Syscall error while processing TLS connection: " + std::string(strerror(errno))); + if (errno == 0) { + throw std::runtime_error("TLS connection closed by remote end"); + } + else { + throw std::runtime_error("Syscall error while processing TLS connection: " + std::string(strerror(errno))); + } } else if (error == SSL_ERROR_ZERO_RETURN) { throw std::runtime_error("TLS connection closed by remote end"); @@ -302,7 +310,7 @@ class OpenSSLTLSConnection: public TLSConnection return IOState::Done; } - IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead) override + IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead, bool allowIncomplete) override { do { int res = SSL_read(d_conn.get(), reinterpret_cast(&buffer.at(pos)), static_cast(toRead - pos)); @@ -311,13 +319,16 @@ class OpenSSLTLSConnection: public TLSConnection } else { pos += static_cast(res); + if (allowIncomplete) { + break; + } } } while (pos < toRead); return IOState::Done; } - size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout) override + size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout, bool allowIncomplete) override { size_t got = 0; struct timeval start = {0, 0}; @@ -333,6 +344,9 @@ class OpenSSLTLSConnection: public TLSConnection } else { got += static_cast(res); + if (allowIncomplete) { + break; + } } if (totalTimeout.tv_sec != 0 || totalTimeout.tv_usec != 0) { @@ -377,6 +391,28 @@ class OpenSSLTLSConnection: public TLSConnection return false; } + bool isUsable() const override + { + if (!d_conn) { + return false; + } + + char buf; + int res = SSL_peek(d_conn.get(), &buf, sizeof(buf)); + if (res > 0) { + return true; + } + try { + convertIORequestToIOState(res); + return true; + } + catch (...) { + return false; + } + + return false; + } + void close() override { if (d_conn) { @@ -395,6 +431,29 @@ class OpenSSLTLSConnection: public TLSConnection return std::string(); } + std::vector getNextProtocol() const override + { + std::vector result; + if (!d_conn) { + return result; + } + + const unsigned char* alpn = nullptr; + unsigned int alpnLen = 0; +#ifdef HAVE_SSL_GET0_NEXT_PROTO_NEGOTIATED + SSL_get0_next_proto_negotiated(d_conn.get(), &alpn, &alpnLen); +#endif +#ifdef HAVE_SSL_GET0_ALPN_SELECTED + if (alpn == nullptr) { + SSL_get0_alpn_selected(d_conn.get(), &alpn, &alpnLen); + } +#endif + if (alpn != nullptr && alpnLen > 0) { + result.insert(result.end(), alpn, alpn + alpnLen); + } + return result; + } + LibsslTLSVersion getTLSVersion() const override { auto proto = SSL_version(d_conn.get()); @@ -422,13 +481,9 @@ class OpenSSLTLSConnection: public TLSConnection return false; } - std::unique_ptr getSession() override + std::vector> getSessions() override { - if (d_tlsSession) { - return std::move(d_tlsSession); - } - - throw std::runtime_error("Unable to get an OpenSSL session"); + return std::move(d_tlsSessions); } void setSession(std::unique_ptr& session) override @@ -446,9 +501,9 @@ class OpenSSLTLSConnection: public TLSConnection native.release(); } - void setNewTicket(SSL_SESSION* session) + void addNewTicket(SSL_SESSION* session) { - d_tlsSession = std::unique_ptr(new OpenSSLSession(std::unique_ptr(session, SSL_SESSION_free))); + d_tlsSessions.push_back(std::make_unique(std::unique_ptr(session, SSL_SESSION_free))); } static int s_tlsConnIndex; @@ -456,9 +511,12 @@ class OpenSSLTLSConnection: public TLSConnection private: static std::atomic_flag s_initTLSConnIndex; + std::vector> d_tlsSessions; + /* server context */ std::shared_ptr d_feContext; + /* client context */ + std::shared_ptr d_tlsCtx; std::unique_ptr d_conn; - std::unique_ptr d_tlsSession{nullptr}; std::string d_hostname; struct timeval d_timeout; }; @@ -470,7 +528,7 @@ class OpenSSLTLSIOCtx: public TLSCtx { public: /* server side context */ - OpenSSLTLSIOCtx(TLSFrontend& fe): d_feContext(std::make_shared(fe.d_addr, fe.d_tlsConfig)), d_tlsCtx(std::unique_ptr(nullptr, SSL_CTX_free)) + OpenSSLTLSIOCtx(TLSFrontend& fe): d_feContext(std::make_shared(fe.d_addr, fe.d_tlsConfig)) { d_ticketsKeyRotationDelay = fe.d_tlsConfig.d_ticketsKeyRotationDelay; @@ -505,7 +563,7 @@ class OpenSSLTLSIOCtx: public TLSCtx } /* client side context */ - OpenSSLTLSIOCtx(const TLSContextParameters& params): d_tlsCtx(std::unique_ptr(nullptr, SSL_CTX_free)) + OpenSSLTLSIOCtx(const TLSContextParameters& params) { int sslOptions = SSL_OP_NO_SSLv2 | @@ -515,13 +573,20 @@ class OpenSSLTLSIOCtx: public TLSCtx SSL_OP_SINGLE_DH_USE | SSL_OP_SINGLE_ECDH_USE | SSL_OP_CIPHER_SERVER_PREFERENCE; + if (!params.d_enableRenegotiation) { +#ifdef SSL_OP_NO_RENEGOTIATION + sslOptions |= SSL_OP_NO_RENEGOTIATION; +#elif defined(SSL_OP_NO_CLIENT_RENEGOTIATION) + sslOptions |= SSL_OP_NO_CLIENT_RENEGOTIATION; +#endif + } registerOpenSSLUser(); #ifdef HAVE_TLS_CLIENT_METHOD - d_tlsCtx = std::unique_ptr(SSL_CTX_new(TLS_client_method()), SSL_CTX_free); + d_tlsCtx = std::shared_ptr(SSL_CTX_new(TLS_client_method()), SSL_CTX_free); #else - d_tlsCtx = std::unique_ptr(SSL_CTX_new(SSLv23_client_method()), SSL_CTX_free); + d_tlsCtx = std::shared_ptr(SSL_CTX_new(SSLv23_client_method()), SSL_CTX_free); #endif if (!d_tlsCtx) { ERR_print_errors_fp(stderr); @@ -569,6 +634,12 @@ class OpenSSLTLSIOCtx: public TLSCtx but we don't want OpenSSL to cache the session itself so we set SSL_SESS_CACHE_NO_INTERNAL_STORE as well */ SSL_CTX_set_session_cache_mode(d_tlsCtx.get(), SSL_SESS_CACHE_CLIENT | SSL_SESS_CACHE_NO_INTERNAL_STORE); SSL_CTX_sess_set_new_cb(d_tlsCtx.get(), &OpenSSLTLSIOCtx::newTicketFromServerCb); + +#ifdef SSL_MODE_RELEASE_BUFFERS + if (params.d_releaseBuffers) { + SSL_CTX_set_mode(d_tlsCtx.get(), SSL_MODE_RELEASE_BUFFERS); + } +#endif } ~OpenSSLTLSIOCtx() override @@ -618,7 +689,7 @@ class OpenSSLTLSIOCtx: public TLSCtx return 0; } - conn->setNewTicket(session); + conn->addNewTicket(session); return 1; } @@ -631,7 +702,7 @@ class OpenSSLTLSIOCtx: public TLSCtx std::unique_ptr getClientConnection(const std::string& host, int socket, const struct timeval& timeout) override { - return std::make_unique(host, socket, timeout, d_tlsCtx.get()); + return std::make_unique(host, socket, timeout, d_tlsCtx); } void rotateTicketsKey(time_t now) override @@ -662,9 +733,74 @@ class OpenSSLTLSIOCtx: public TLSCtx return "openssl"; } + bool setALPNProtos(const std::vector>& protos) override + { + if (d_feContext && d_feContext->d_tlsCtx) { + d_alpnProtos = protos; + libssl_set_alpn_select_callback(d_feContext->d_tlsCtx.get(), alpnServerSelectCallback, this); + return true; + } + if (d_tlsCtx) { + return libssl_set_alpn_protos(d_tlsCtx.get(), protos); + } + return false; + } + + bool setNextProtocolSelectCallback(bool(*cb)(unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen)) override + { + d_nextProtocolSelectCallback = cb; + libssl_set_npn_select_callback(d_tlsCtx.get(), npnSelectCallback, this); + return true; + } + private: - std::shared_ptr d_feContext; - std::unique_ptr d_tlsCtx; // client context + /* called in a client context, if the client advertised more than one ALPN values and the server returned more than one as well, to select the one to use. */ + static int npnSelectCallback(SSL* s, unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen, void* arg) + { + if (!arg) { + return SSL_TLSEXT_ERR_ALERT_WARNING; + } + OpenSSLTLSIOCtx* obj = reinterpret_cast(arg); + if (obj->d_nextProtocolSelectCallback) { + return (*obj->d_nextProtocolSelectCallback)(out, outlen, in, inlen) ? SSL_TLSEXT_ERR_OK : SSL_TLSEXT_ERR_ALERT_WARNING; + } + + return SSL_TLSEXT_ERR_OK; + } + + static int alpnServerSelectCallback(SSL*, const unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen, void* arg) + { + if (!arg) { + return SSL_TLSEXT_ERR_ALERT_WARNING; + } + OpenSSLTLSIOCtx* obj = reinterpret_cast(arg); + + size_t pos = 0; + while (pos < inlen) { + size_t protoLen = in[pos]; + pos++; + if (protoLen > (inlen - pos)) { + /* something is very wrong */ + return SSL_TLSEXT_ERR_ALERT_WARNING; + } + + for (const auto& tentative : obj->d_alpnProtos) { + if (tentative.size() == protoLen && memcmp(in + pos, tentative.data(), tentative.size()) == 0) { + *out = in + pos; + *outlen = protoLen; + return SSL_TLSEXT_ERR_OK; + } + } + pos += protoLen; + } + + return SSL_TLSEXT_ERR_NOACK; + } + + std::vector> d_alpnProtos; // store the supported ALPN protocols, so that the server can select based on what the client sent + std::shared_ptr d_feContext{nullptr}; + std::shared_ptr d_tlsCtx{nullptr}; // client context, on a server-side the context is stored in d_feContext->d_tlsCtx + bool (*d_nextProtocolSelectCallback)(unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen){nullptr}; }; #endif /* HAVE_LIBSSL */ @@ -796,7 +932,7 @@ class GnuTLSConnection: public TLSConnection { public: /* server side connection */ - GnuTLSConnection(int socket, const struct timeval& timeout, const gnutls_certificate_credentials_t creds, const gnutls_priority_t priorityCache, std::shared_ptr& ticketsKey, bool enableTickets): d_ticketsKey(ticketsKey), d_conn(std::unique_ptr(nullptr, gnutls_deinit)) + GnuTLSConnection(int socket, const struct timeval& timeout, std::shared_ptr& creds, const gnutls_priority_t priorityCache, std::shared_ptr& ticketsKey, bool enableTickets): d_creds(creds), d_ticketsKey(ticketsKey), d_conn(std::unique_ptr(nullptr, gnutls_deinit)) { unsigned int sslOptions = GNUTLS_SERVER | GNUTLS_NONBLOCK; #ifdef GNUTLS_NO_SIGNAL @@ -813,7 +949,7 @@ class GnuTLSConnection: public TLSConnection d_conn = std::unique_ptr(conn, gnutls_deinit); conn = nullptr; - if (gnutls_credentials_set(d_conn.get(), GNUTLS_CRD_CERTIFICATE, creds) != GNUTLS_E_SUCCESS) { + if (gnutls_credentials_set(d_conn.get(), GNUTLS_CRD_CERTIFICATE, d_creds.get()) != GNUTLS_E_SUCCESS) { throw std::runtime_error("Error setting certificate and key to TLS connection"); } @@ -836,7 +972,7 @@ class GnuTLSConnection: public TLSConnection } /* client-side connection */ - GnuTLSConnection(const std::string& host, int socket, const struct timeval& timeout, const gnutls_certificate_credentials_t creds, const gnutls_priority_t priorityCache, bool validateCerts): d_conn(std::unique_ptr(nullptr, gnutls_deinit)), d_host(host), d_client(true) + GnuTLSConnection(const std::string& host, int socket, const struct timeval& timeout, std::shared_ptr& creds, const gnutls_priority_t priorityCache, bool validateCerts): d_creds(creds), d_conn(std::unique_ptr(nullptr, gnutls_deinit)), d_host(host), d_client(true) { unsigned int sslOptions = GNUTLS_CLIENT | GNUTLS_NONBLOCK; #ifdef GNUTLS_NO_SIGNAL @@ -853,7 +989,7 @@ class GnuTLSConnection: public TLSConnection d_conn = std::unique_ptr(conn, gnutls_deinit); conn = nullptr; - int rc = gnutls_credentials_set(d_conn.get(), GNUTLS_CRD_CERTIFICATE, creds); + int rc = gnutls_credentials_set(d_conn.get(), GNUTLS_CRD_CERTIFICATE, d_creds.get()); if (rc != GNUTLS_E_SUCCESS) { throw std::runtime_error("Error setting certificate and key to TLS connection: " + std::string(gnutls_strerror(rc))); } @@ -908,7 +1044,7 @@ class GnuTLSConnection: public TLSConnection if (ret != GNUTLS_E_SUCCESS || sess.size <= 4) { throw std::runtime_error("Error getting GnuTLSSession: " + std::string(gnutls_strerror(ret))); } - conn->d_tlsSession = std::unique_ptr(new GnuTLSSession(sess)); + conn->d_tlsSessions.push_back(std::make_unique(sess)); return 0; } @@ -1075,7 +1211,7 @@ class GnuTLSConnection: public TLSConnection return IOState::Done; } - IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead) override + IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead, bool allowIncomplete) override { if (!d_handshakeDone) { /* As opposed to OpenSSL, GnuTLS will not transparently finish the handshake for us, @@ -1093,6 +1229,9 @@ class GnuTLSConnection: public TLSConnection } else if (res > 0) { pos += static_cast(res); + if (allowIncomplete) { + break; + } } else if (res < 0) { if (gnutls_error_is_fatal(res)) { @@ -1108,7 +1247,7 @@ class GnuTLSConnection: public TLSConnection return IOState::Done; } - size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout) override + size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout, bool allowIncomplete) override { size_t got = 0; struct timeval start{0,0}; @@ -1124,6 +1263,9 @@ class GnuTLSConnection: public TLSConnection } else if (res > 0) { got += static_cast(res); + if (allowIncomplete) { + break; + } } else if (res < 0) { if (gnutls_error_is_fatal(res)) { @@ -1197,6 +1339,16 @@ class GnuTLSConnection: public TLSConnection return false; } + bool isUsable() const override + { + if (!d_conn) { + return false; + } + + /* as far as I can tell we can't peek so we cannot do better */ + return isTCPSocketUsable(d_socket); + } + std::string getServerNameIndication() const override { if (d_conn) { @@ -1214,6 +1366,20 @@ class GnuTLSConnection: public TLSConnection return std::string(); } + std::vector getNextProtocol() const override + { + std::vector result; + if (!d_conn) { + return result; + } + gnutls_datum_t next; + if (gnutls_alpn_get_selected_protocol(d_conn.get(), &next) != GNUTLS_E_SUCCESS) { + return result; + } + result.insert(result.end(), next.data, next.data + next.size); + return result; + } + LibsslTLSVersion getTLSVersion() const override { auto proto = gnutls_protocol_get_version(d_conn.get()); @@ -1241,13 +1407,9 @@ class GnuTLSConnection: public TLSConnection return false; } - std::unique_ptr getSession() override + std::vector> getSessions() override { - if (d_tlsSession) { - return std::move(d_tlsSession); - } - - throw std::runtime_error("No GnuTLSSession available yet"); + return std::move(d_tlsSessions); } void setSession(std::unique_ptr& session) override @@ -1273,10 +1435,30 @@ class GnuTLSConnection: public TLSConnection } } + bool setALPNProtos(const std::vector>& protos) + { + std::vector values; + values.reserve(protos.size()); + for (const auto& proto : protos) { + gnutls_datum_t value; + value.data = const_cast(proto.data()); + value.size = proto.size(); + values.push_back(value); + } + unsigned int flags = 0; +#if GNUTLS_VERSION_NUMBER >= 0x030500 + flags |= GNUTLS_ALPN_MANDATORY; +#elif defined(GNUTLS_ALPN_MAND) + flags |= GNUTLS_ALPN_MAND; +#endif + return gnutls_alpn_set_protocols(d_conn.get(), values.data(), values.size(), flags); + } + private: + std::shared_ptr d_creds; std::shared_ptr d_ticketsKey; std::unique_ptr d_conn; - std::unique_ptr d_tlsSession{nullptr}; + std::vector> d_tlsSessions; std::string d_host; bool d_client{false}; bool d_handshakeDone{false}; @@ -1286,7 +1468,7 @@ class GnuTLSIOCtx: public TLSCtx { public: /* server side context */ - GnuTLSIOCtx(TLSFrontend& fe): d_creds(std::unique_ptr(nullptr, gnutls_certificate_free_credentials)), d_enableTickets(fe.d_tlsConfig.d_enableTickets) + GnuTLSIOCtx(TLSFrontend& fe): d_enableTickets(fe.d_tlsConfig.d_enableTickets) { int rc = 0; d_ticketsKeyRotationDelay = fe.d_tlsConfig.d_ticketsKeyRotationDelay; @@ -1297,7 +1479,7 @@ class GnuTLSIOCtx: public TLSCtx throw std::runtime_error("Error allocating credentials for TLS context on " + fe.d_addr.toStringWithPort() + ": " + gnutls_strerror(rc)); } - d_creds = std::unique_ptr(creds, gnutls_certificate_free_credentials); + d_creds = std::shared_ptr(creds, gnutls_certificate_free_credentials); creds = nullptr; for (const auto& pair : fe.d_tlsConfig.d_certKeyPairs) { @@ -1342,7 +1524,7 @@ class GnuTLSIOCtx: public TLSCtx } /* client side context */ - GnuTLSIOCtx(const TLSContextParameters& params): d_creds(std::unique_ptr(nullptr, gnutls_certificate_free_credentials)), d_enableTickets(true), d_validateCerts(params.d_validateCertificates) + GnuTLSIOCtx(const TLSContextParameters& params): d_contextParameters(std::make_unique(params)), d_enableTickets(true), d_validateCerts(params.d_validateCertificates) { int rc = 0; @@ -1352,7 +1534,7 @@ class GnuTLSIOCtx: public TLSCtx throw std::runtime_error("Error allocating credentials for TLS context: " + std::string(gnutls_strerror(rc))); } - d_creds = std::unique_ptr(creds, gnutls_certificate_free_credentials); + d_creds = std::shared_ptr(creds, gnutls_certificate_free_credentials); creds = nullptr; if (params.d_validateCertificates) { @@ -1394,12 +1576,53 @@ class GnuTLSIOCtx: public TLSCtx ticketsKey = *(d_ticketsKey.read_lock()); } - return std::make_unique(socket, timeout, d_creds.get(), d_priorityCache, ticketsKey, d_enableTickets); + auto connection = std::make_unique(socket, timeout, d_creds, d_priorityCache, ticketsKey, d_enableTickets); + if (!d_protos.empty()) { + connection->setALPNProtos(d_protos); + } + return connection; + } + + static std::shared_ptr getPerThreadCredentials(bool validate, const std::string& caStore) + { + static thread_local std::map, std::shared_ptr> t_credentials; + auto& entry = t_credentials[{validate, caStore}]; + if (!entry) { + gnutls_certificate_credentials_t creds; + int rc = gnutls_certificate_allocate_credentials(&creds); + if (rc != GNUTLS_E_SUCCESS) { + throw std::runtime_error("Error allocating credentials for TLS context: " + std::string(gnutls_strerror(rc))); + } + + entry = std::shared_ptr(creds, gnutls_certificate_free_credentials); + creds = nullptr; + + if (validate) { + if (caStore.empty()) { + rc = gnutls_certificate_set_x509_system_trust(entry.get()); + if (rc < 0) { + throw std::runtime_error("Error adding the system's default trusted CAs: " + std::string(gnutls_strerror(rc))); + } + } + else { + rc = gnutls_certificate_set_x509_trust_file(entry.get(), caStore.c_str(), GNUTLS_X509_FMT_PEM); + if (rc < 0) { + throw std::runtime_error("Error adding '" + caStore + "' to the trusted CAs: " + std::string(gnutls_strerror(rc))); + } + } + } + } + return entry; } std::unique_ptr getClientConnection(const std::string& host, int socket, const struct timeval& timeout) override { - return std::make_unique(host, socket, timeout, d_creds.get(), d_priorityCache, d_validateCerts); + auto creds = getPerThreadCredentials(d_contextParameters->d_validateCertificates, d_contextParameters->d_caStore); + auto connection = std::make_unique(host, socket, timeout, creds, d_priorityCache, d_validateCerts); + if (!d_protos.empty()) { + connection->setALPNProtos(d_protos); + } + return connection; } void rotateTicketsKey(time_t now) override @@ -1445,8 +1668,21 @@ class GnuTLSIOCtx: public TLSCtx return "gnutls"; } + bool setALPNProtos(const std::vector>& protos) override + { +#ifdef HAVE_GNUTLS_ALPN_SET_PROTOCOLS + d_protos = protos; + return true; +#else + return false; +#endif + } + private: - std::unique_ptr d_creds; + /* client context parameters */ + std::unique_ptr d_contextParameters{nullptr}; + std::shared_ptr d_creds; + std::vector> d_protos; gnutls_priority_t d_priorityCache{nullptr}; SharedLockGuarded> d_ticketsKey{nullptr}; bool d_enableTickets{true}; @@ -1457,6 +1693,17 @@ class GnuTLSIOCtx: public TLSCtx #endif /* HAVE_DNS_OVER_TLS */ +bool setupDoTProtocolNegotiation(std::shared_ptr& ctx) +{ + if (ctx == nullptr) { + return false; + } + /* we want to set the ALPN to dot (RFC7858), if only to mitigate the ALPACA attack */ + const std::vector> dotAlpns = {{'d', 'o', 't'}}; + ctx->setALPNProtos(dotAlpns); + return true; +} + bool TLSFrontend::setupTLS() { #ifdef HAVE_DNS_OVER_TLS @@ -1466,6 +1713,7 @@ bool TLSFrontend::setupTLS() #ifdef HAVE_GNUTLS if (d_provider == "gnutls") { newCtx = std::make_shared(*this); + setupDoTProtocolNegotiation(newCtx); std::atomic_store_explicit(&d_ctx, newCtx, std::memory_order_release); return true; } @@ -1473,6 +1721,7 @@ bool TLSFrontend::setupTLS() #ifdef HAVE_LIBSSL if (d_provider == "openssl") { newCtx = std::make_shared(*this); + setupDoTProtocolNegotiation(newCtx); std::atomic_store_explicit(&d_ctx, newCtx, std::memory_order_release); return true; } @@ -1486,6 +1735,7 @@ bool TLSFrontend::setupTLS() #endif /* HAVE_GNUTLS */ #endif /* HAVE_LIBSSL */ + setupDoTProtocolNegotiation(newCtx); std::atomic_store_explicit(&d_ctx, newCtx, std::memory_order_release); #endif /* HAVE_DNS_OVER_TLS */ return true; diff --git a/pdns/tcpiohandler.hh b/pdns/tcpiohandler.hh index d18a92b05af2..5b23e00dc3b7 100644 --- a/pdns/tcpiohandler.hh +++ b/pdns/tcpiohandler.hh @@ -27,16 +27,18 @@ public: virtual IOState tryConnect(bool fastOpen, const ComboAddress& remote) = 0; virtual void connect(bool fastOpen, const ComboAddress& remote, const struct timeval& timeout) = 0; virtual IOState tryHandshake() = 0; - virtual size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout={0,0}) = 0; + virtual size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout={0,0}, bool allowIncomplete=false) = 0; virtual size_t write(const void* buffer, size_t bufferSize, const struct timeval& writeTimeout) = 0; virtual IOState tryWrite(const PacketBuffer& buffer, size_t& pos, size_t toWrite) = 0; - virtual IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead) = 0; + virtual IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead, bool allowIncomplete=false) = 0; virtual bool hasBufferedData() const = 0; virtual std::string getServerNameIndication() const = 0; + virtual std::vector getNextProtocol() const = 0; virtual LibsslTLSVersion getTLSVersion() const = 0; virtual bool hasSessionBeenResumed() const = 0; - virtual std::unique_ptr getSession() = 0; + virtual std::vector> getSessions() = 0; virtual void setSession(std::unique_ptr& session) = 0; + virtual bool isUsable() const = 0; virtual void close() = 0; void setUnknownTicketKey() @@ -111,6 +113,18 @@ public: virtual size_t getTicketsKeysCount() = 0; virtual std::string getName() const = 0; + /* set the advertised ALPN protocols, in client or server context */ + virtual bool setALPNProtos(const std::vector>& protos) + { + return false; + } + + /* called in a client context, if the client advertised more than one ALPN values and the server returned more than one as well, to select the one to use. */ + virtual bool setNextProtocolSelectCallback(bool(*)(unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen)) + { + return false; + } + protected: std::atomic_flag d_rotatingTicketsKey; std::atomic d_ticketsKeyNextRotation{0}; @@ -274,10 +288,14 @@ public: d_fastOpen = true; } else { - SConnectWithTimeout(d_socket, remote, /* no timeout, we will handle it ourselves */ timeval{0,0}); + if (!s_disableConnectForUnitTests) { + SConnectWithTimeout(d_socket, remote, /* no timeout, we will handle it ourselves */ timeval{0,0}); + } } #else - SConnectWithTimeout(d_socket, remote, /* no timeout, we will handle it ourselves */ timeval{0,0}); + if (!s_disableConnectForUnitTests) { + SConnectWithTimeout(d_socket, remote, /* no timeout, we will handle it ourselves */ timeval{0,0}); + } #endif /* MSG_FASTOPEN */ if (d_conn) { @@ -306,10 +324,14 @@ public: d_fastOpen = true; } else { - SConnectWithTimeout(d_socket, remote, timeout); + if (!s_disableConnectForUnitTests) { + SConnectWithTimeout(d_socket, remote, timeout); + } } #else - SConnectWithTimeout(d_socket, remote, timeout); + if (!s_disableConnectForUnitTests) { + SConnectWithTimeout(d_socket, remote, timeout); + } #endif /* MSG_FASTOPEN */ if (d_conn) { @@ -325,12 +347,12 @@ public: return IOState::Done; } - size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout = {0,0}) + size_t read(void* buffer, size_t bufferSize, const struct timeval& readTimeout, const struct timeval& totalTimeout = {0,0}, bool allowIncomplete=false) { if (d_conn) { - return d_conn->read(buffer, bufferSize, readTimeout, totalTimeout); + return d_conn->read(buffer, bufferSize, readTimeout, totalTimeout, allowIncomplete); } else { - return readn2WithTimeout(d_socket, buffer, bufferSize, readTimeout, totalTimeout); + return readn2WithTimeout(d_socket, buffer, bufferSize, readTimeout, totalTimeout, allowIncomplete); } } @@ -340,14 +362,14 @@ public: return Done when toRead bytes have been read, needRead or needWrite if the IO operation would block. */ - IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead) + IOState tryRead(PacketBuffer& buffer, size_t& pos, size_t toRead, bool allowIncomplete=false) { if (buffer.size() < toRead || pos >= toRead) { throw std::out_of_range("Calling tryRead() with a too small buffer (" + std::to_string(buffer.size()) + ") for a read of " + std::to_string(toRead - pos) + " bytes starting at " + std::to_string(pos)); } if (d_conn) { - return d_conn->tryRead(buffer, pos, toRead); + return d_conn->tryRead(buffer, pos, toRead, allowIncomplete); } do { @@ -365,6 +387,9 @@ public: } pos += static_cast(res); + if (allowIncomplete) { + break; + } } while (pos < toRead); @@ -462,6 +487,14 @@ public: return std::string(); } + std::vector getNextProtocol() const + { + if (d_conn) { + return d_conn->getNextProtocol(); + } + return std::vector(); + } + LibsslTLSVersion getTLSVersion() const { if (d_conn) { @@ -497,15 +530,25 @@ public: } } - std::unique_ptr getTLSSession() + std::vector> getTLSSessions() { if (!d_conn) { - throw std::runtime_error("Trying to get a TLS session from a non-TLS handler"); + throw std::runtime_error("Trying to get TLS sessions from a non-TLS handler"); } - return d_conn->getSession(); + return d_conn->getSessions(); } + bool isUsable() const + { + if (!d_conn) { + return isTCPSocketUsable(d_socket); + } + return d_conn->isUsable(); + } + + const static bool s_disableConnectForUnitTests; + private: std::unique_ptr d_conn{nullptr}; ComboAddress d_remote; @@ -522,6 +565,9 @@ struct TLSContextParameters std::string d_ciphers13; std::string d_caStore; bool d_validateCertificates{true}; + bool d_releaseBuffers{true}; + bool d_enableRenegotiation{false}; }; std::shared_ptr getTLSContext(const TLSContextParameters& params); +bool setupDoTProtocolNegotiation(std::shared_ptr& ctx); diff --git a/pdns/tcpreceiver.cc b/pdns/tcpreceiver.cc index b9c619eb2d77..9670c9a8bc4f 100644 --- a/pdns/tcpreceiver.cc +++ b/pdns/tcpreceiver.cc @@ -58,6 +58,8 @@ #include "namespaces.hh" #include "signingpipe.hh" #include "stubresolver.hh" +#include "proxy-protocol.hh" +#include "noinitvector.hh" extern AuthPacketCache PC; extern StatBag S; @@ -66,24 +68,22 @@ extern StatBag S; \brief This file implements the tcpreceiver that receives and answers questions over TCP/IP */ -std::mutex TCPNameserver::s_plock; std::unique_ptr TCPNameserver::d_connectionroom_sem{nullptr}; -std::unique_ptr TCPNameserver::s_P{nullptr}; +LockGuarded> TCPNameserver::s_P{nullptr}; unsigned int TCPNameserver::d_maxTCPConnections = 0; NetmaskGroup TCPNameserver::d_ng; size_t TCPNameserver::d_maxTransactionsPerConn; size_t TCPNameserver::d_maxConnectionsPerClient; unsigned int TCPNameserver::d_idleTimeout; unsigned int TCPNameserver::d_maxConnectionDuration; -std::mutex TCPNameserver::s_clientsCountMutex; -std::map TCPNameserver::s_clientsCount; +LockGuarded> TCPNameserver::s_clientsCount; void TCPNameserver::go() { g_log<reset(); try { - s_P=make_unique(); + *(s_P.lock()) = make_unique(); } catch(PDNSException &ae) { g_log< lock(s_clientsCountMutex); - s_clientsCount[remote]--; - if (s_clientsCount[remote] == 0) { - s_clientsCount.erase(remote); + auto count = s_clientsCount.lock(); + auto it = count->find(remote); + if (it == count->end()) { + // this is worrying, but nothing we can do at this point + return; + } + --it->second; + if (it->second == 0) { + count->erase(it); } } } @@ -240,9 +245,59 @@ void TCPNameserver::doConnection(int fd) try { int mesgsize=65535; boost::scoped_array mesg(new char[mesgsize]); - + std::optional inner_remote; + bool inner_tcp = false; + DLOG(g_log<<"TCP Connection accepted on fd "<(used) > g_proxyProtocolMaximumSize) { + throw NetworkError("Error reading PROXYv2 header from TCP client "+remote.toString()+": PROXYv2 header too big"); + } + else { // used > 0 && used <= g_proxyProtocolMaximumSize + break; + } + } + ComboAddress psource, pdestination; + bool proxyProto, tcp; + std::vector ppvalues; + + used = parseProxyHeader(proxyData, proxyProto, psource, pdestination, tcp, ppvalues); + if (used <= 0) { + throw NetworkError("Error reading PROXYv2 header from TCP client "+remote.toString()+": PROXYv2 header was invalid"); + } + if (static_cast(used) > g_proxyProtocolMaximumSize) { + throw NetworkError("Error reading PROXYv2 header from TCP client "+remote.toString()+": PROXYv2 header was oversized"); + } + inner_remote = psource; + inner_tcp = tcp; + } + for(;;) { unsigned int remainingTime = 0; transactions++; @@ -288,10 +343,17 @@ void TCPNameserver::doConnection(int fd) packet=make_unique(true); packet->setRemote(&remote); packet->d_tcp=true; + if (inner_remote) { + packet->d_inner_remote = inner_remote; + packet->d_tcp = inner_tcp; + } packet->setSocket(fd); if(packet->parse(mesg.get(), pktlen)<0) break; - + + if (packet->hasEDNSCookie()) + S.inc("tcp-cookie-queries"); + if(packet->qtype.getCode()==QType::AXFR) { doAXFR(packet->qdomain, packet, fd); continue; @@ -305,12 +367,7 @@ void TCPNameserver::doConnection(int fd) std::unique_ptr reply; auto cached = make_unique(false); if(logDNSQueries) { - string remote_text; - if(packet->hasEDNSSubnet()) - remote_text = packet->getRemote().toString() + "<-" + packet->getRealRemote().toString(); - else - remote_text = packet->getRemote().toString(); - g_log << Logger::Notice<<"TCP Remote "<< remote_text <<" wants '" << packet->qdomain<<"|"<qtype.toString() << + g_log << Logger::Notice<<"TCP Remote "<< packet->getRemoteString() <<" wants '" << packet->qdomain<<"|"<qtype.toString() << "', do = " <d_dnssecOk <<", bufsize = "<< packet->getMaxReplyLen(); } @@ -334,13 +391,13 @@ void TCPNameserver::doConnection(int fd) } } { - std::lock_guard l(s_plock); - if(!s_P) { + auto packetHandler = s_P.lock(); + if (!*packetHandler) { g_log<(); + *packetHandler = make_unique(); } - reply= s_P->doQuestion(*packet); // we really need to ask the backend :-) + reply = (*packetHandler)->doQuestion(*packet); // we really need to ask the backend :-) } if(!reply) // unable to write an answer? @@ -350,8 +407,7 @@ void TCPNameserver::doConnection(int fd) } } catch(PDNSException &ae) { - std::lock_guard l(s_plock); - s_P.reset(); // on next call, backend will be recycled + s_P.lock()->reset(); // on next call, backend will be recycled g_log<& q, bool isAXFR) +bool TCPNameserver::canDoAXFR(std::unique_ptr& q, bool isAXFR, std::unique_ptr& packetHandler) { if(::arg().mustDo("disable-axfr")) return false; - string logPrefix=string(isAXFR ? "A" : "I")+"XFR-out zone '"+q->qdomain.toLogString()+"', client '"+q->getRemote().toStringWithPort()+"', "; + string logPrefix=string(isAXFR ? "A" : "I")+"XFR-out zone '"+q->qdomain.toLogString()+"', client '"+q->getInnerRemote().toStringWithPort()+"', "; if(q->d_havetsig) { // if you have one, it must be good TSIGRecordContent trc; DNSName keyname; string secret; - if(!q->checkForCorrectTSIG(s_P->getBackend(), &keyname, &secret, &trc)) { + if(!q->checkForCorrectTSIG(packetHandler->getBackend(), &keyname, &secret, &trc)) { return false; } else { getTSIGHashEnum(trc.d_algoName, q->d_tsig_algo); } - DNSSECKeeper dk(s_P->getBackend()); + DNSSECKeeper dk(packetHandler->getBackend()); if(!dk.TSIGGrantsAccess(q->qdomain, keyname)) { g_log<d_tsig_algo)<<"' does not grant access"<& q, bool isAXFR) } // cerr<<"checking allow-axfr-ips"<d_remote )) { + if(!(::arg()["allow-axfr-ips"].empty()) && d_ng.match( q->getInnerRemote() )) { g_log<& q, bool isAXFR) // cerr<<"doing per-zone-axfr-acls"<getBackend()->getSOAUncached(q->qdomain,sd)) { + if(packetHandler->getBackend()->getSOAUncached(q->qdomain,sd)) { // cerr<<"got backend and SOA"< acl; - s_P->getBackend()->getDomainMetadata(q->qdomain, "ALLOW-AXFR-FROM", acl); + packetHandler->getBackend()->getDomainMetadata(q->qdomain, "ALLOW-AXFR-FROM", acl); for (const auto & i : acl) { // cerr<<"matching against "<<*i<& q, bool isAXFR) nsset.insert(DNSName(rr.content)); } for(const auto & j: nsset) { - vector nsips=fns.lookup(j, s_P->getBackend()); + vector nsips=fns.lookup(j, packetHandler->getBackend()); for(const auto & nsip : nsips) { // cerr<<"got "<<*k<<" from AUTO-NS"<getRemote().toString()) + if(nsip == q->getInnerRemote().toString()) { // cerr<<"got AUTO-NS hit"<& q, bool isAXFR) else { Netmask nm = Netmask(i); - if(nm.match( (ComboAddress *) &q->d_remote )) + if(nm.match( q->getInnerRemote() )) { g_log<& q, bool isAXFR) extern CommunicatorClass Communicator; - if(Communicator.justNotified(q->qdomain, q->getRemote().toString())) { // we just notified this ip + if(Communicator.justNotified(q->qdomain, q->getInnerRemote().toString())) { // we just notified this ip g_log<& q, int outsock) { - string logPrefix="AXFR-out zone '"+target.toLogString()+"', client '"+q->getRemote().toStringWithPort()+"', "; + string logPrefix="AXFR-out zone '"+target.toLogString()+"', client '"+q->getRemoteString()+"', "; std::unique_ptr outpacket= getFreshAXFRPacket(q); if(q->d_dnssecOk) @@ -502,22 +557,22 @@ int TCPNameserver::doAXFR(const DNSName &target, std::unique_ptr& q, // determine if zone exists and AXFR is allowed using existing backend before spawning a new backend. SOAData sd; { - std::lock_guard l(s_plock); + auto packetHandler = s_P.lock(); DLOG(g_log<(); + *packetHandler = make_unique(); } // canDoAXFR does all the ACL checks, and has the if(disable-axfr) shortcut, call it first. - if (!canDoAXFR(q, true)) { + if (!canDoAXFR(q, true, *packetHandler)) { g_log<setRcode(RCode::NotAuth); sendPacket(outpacket,outsock); return 0; } - if(!s_P->getBackend()->getSOAUncached(target, sd)) { + if(!(*packetHandler)->getBackend()->getSOAUncached(target, sd)) { g_log<setRcode(RCode::NotAuth); sendPacket(outpacket,outsock); @@ -1023,7 +1078,7 @@ int TCPNameserver::doAXFR(const DNSName &target, std::unique_ptr& q, int TCPNameserver::doIXFR(std::unique_ptr& q, int outsock) { - string logPrefix="IXFR-out zone '"+q->qdomain.toLogString()+"', client '"+q->getRemote().toStringWithPort()+"', "; + string logPrefix="IXFR-out zone '"+q->qdomain.toLogString()+"', client '"+q->getRemoteString()+"', "; std::unique_ptr outpacket=getFreshAXFRPacket(q); if(q->d_dnssecOk) @@ -1067,22 +1122,22 @@ int TCPNameserver::doIXFR(std::unique_ptr& q, int outsock) bool securedZone; bool serialPermitsIXFR; { - std::lock_guard l(s_plock); + auto packetHandler = s_P.lock(); DLOG(g_log<(); + *packetHandler = make_unique(); } // canDoAXFR does all the ACL checks, and has the if(disable-axfr) shortcut, call it first. - if(!canDoAXFR(q, false) || !s_P->getBackend()->getSOAUncached(q->qdomain, sd)) { + if(!canDoAXFR(q, false, *packetHandler) || !(*packetHandler)->getBackend()->getSOAUncached(q->qdomain, sd)) { g_log<setRcode(RCode::NotAuth); sendPacket(outpacket,outsock); return 0; } - DNSSECKeeper dk(s_P->getBackend()); + DNSSECKeeper dk((*packetHandler)->getBackend()); DNSSECKeeper::clearCaches(q->qdomain); bool narrow; securedZone = dk.isSecuredZone(q->qdomain); @@ -1260,13 +1315,13 @@ void TCPNameserver::thread() } else { if (d_maxConnectionsPerClient) { - std::lock_guard lock(s_clientsCountMutex); - if (s_clientsCount[remote] >= d_maxConnectionsPerClient) { + auto clientsCount = s_clientsCount.lock(); + if ((*clientsCount)[remote] >= d_maxConnectionsPerClient) { g_log<wait(); // blocks if no connections are available diff --git a/pdns/tcpreceiver.hh b/pdns/tcpreceiver.hh index 366921a91e56..e7ce8067ed34 100644 --- a/pdns/tcpreceiver.hh +++ b/pdns/tcpreceiver.hh @@ -25,7 +25,6 @@ #include "dnsbackend.hh" #include "packethandler.hh" #include -#include #include #include #include @@ -37,6 +36,7 @@ #include #include +#include "lock.hh" #include "namespaces.hh" class TCPNameserver @@ -49,18 +49,15 @@ public: private: static void sendPacket(std::unique_ptr& p, int outsock, bool last=true); - static int readLength(int fd, ComboAddress *remote); static void getQuestion(int fd, char *mesg, int pktlen, const ComboAddress& remote, unsigned int totalTime); static int doAXFR(const DNSName &target, std::unique_ptr& q, int outsock); static int doIXFR(std::unique_ptr& q, int outsock); - static bool canDoAXFR(std::unique_ptr& q, bool isAXFR); + static bool canDoAXFR(std::unique_ptr& q, bool isAXFR, std::unique_ptr& packetHandler); static void doConnection(int fd); static void decrementClientCount(const ComboAddress& remote); void thread(void); - static std::mutex s_plock; - static std::mutex s_clientsCountMutex; - static std::map s_clientsCount; - static std::unique_ptr s_P; + static LockGuarded> s_clientsCount; + static LockGuarded> s_P; static std::unique_ptr d_connectionroom_sem; static unsigned int d_maxTCPConnections; static NetmaskGroup d_ng; diff --git a/pdns/test-credentials_cc.cc b/pdns/test-credentials_cc.cc new file mode 100644 index 000000000000..89bd6e4a2276 --- /dev/null +++ b/pdns/test-credentials_cc.cc @@ -0,0 +1,117 @@ + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_NO_MAIN + +#include +#include + +#include "config.h" +#include "credentials.hh" + +BOOST_AUTO_TEST_SUITE(credentials_cc) + +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT +BOOST_AUTO_TEST_CASE(test_CredentialsUtils) +{ + const std::string plaintext("test"); + /* generated with hashPassword("test") */ + const std::string sampleHash("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI="); + + auto hashed = hashPassword(plaintext); + BOOST_CHECK(!hashed.empty()); + + BOOST_CHECK(verifyPassword(hashed, plaintext)); + BOOST_CHECK(verifyPassword(sampleHash, plaintext)); + + BOOST_CHECK(!verifyPassword(hashed, "not test")); + BOOST_CHECK(!verifyPassword(sampleHash, "not test")); + + BOOST_CHECK(isPasswordHashed(hashed)); + BOOST_CHECK(isPasswordHashed(sampleHash)); + BOOST_CHECK(!isPasswordHashed(plaintext)); + + { + // hash password with custom parameters + auto customParams = hashPassword(plaintext, 512, 2, 16); + // check that the output is OK + BOOST_CHECK(boost::starts_with(customParams, "$scrypt$ln=9,p=2,r=16$")); + // check that we can verify the password + BOOST_CHECK(verifyPassword(customParams, plaintext)); + } + + // empty + BOOST_CHECK(!isPasswordHashed("")); + // missing leading $ + BOOST_CHECK(!isPasswordHashed("scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // unknown algo + BOOST_CHECK(!isPasswordHashed("$tcrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // missing parameters + BOOST_CHECK(!isPasswordHashed("$scrypt$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // empty parameters + BOOST_CHECK(!isPasswordHashed("$scrypt$$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // missing r + BOOST_CHECK(!isPasswordHashed("$scrypt$ln=10,p=1$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // salt is too short + BOOST_CHECK(!isPasswordHashed("$scrypt$ln=10,p=1,r=8$dGVzdA==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // hash is too short + BOOST_CHECK(!isPasswordHashed("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$c2hvcnQ=")); + // missing salt + BOOST_CHECK(!isPasswordHashed("$scrypt$ln=10,p=1,r=8$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // missing $ between the salt and hash + BOOST_CHECK(!isPasswordHashed("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=")); + // no hash + BOOST_CHECK(!isPasswordHashed("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$")); + // hash is too long + BOOST_CHECK(!isPasswordHashed("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$dGhpcyBpcyBhIHZlcnkgbG9uZyBoYXNoLCBtdWNoIG11Y2ggbG9uZ2VyIHRoYW4gdGhlIG9uZXMgd2UgYXJlIGdlbmVyYXRpbmc=")); + + // empty r + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=10,p=1,r=$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=", plaintext), std::runtime_error); + // too many parameters + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=10,p=1,r=8,t=1$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=", plaintext), std::runtime_error); + // invalid ln + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=A,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=", plaintext), std::runtime_error); + // invalid p + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=10,p=p,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=", plaintext), std::runtime_error); + // work factor is too large + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=16,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=", plaintext), std::runtime_error); + // salt is too long + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=10,p=1,r=8$dGhpcyBpcyBhIHZlcnkgbG9uZyBzYWx0$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=", plaintext), std::runtime_error); + // invalid b64 salt + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI=", plaintext), std::runtime_error); + // invalid b64 hash + BOOST_CHECK_THROW(verifyPassword("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJd", plaintext), std::runtime_error); +} +#endif + +BOOST_AUTO_TEST_CASE(test_CredentialsHolder) +{ + const std::string plaintext("test"); + + auto holder = CredentialsHolder(std::string(plaintext), false); + + BOOST_CHECK(holder.matches(plaintext)); + BOOST_CHECK(!holder.matches("not test")); + BOOST_CHECK(!holder.wasHashed()); + BOOST_CHECK(!holder.isHashed()); + +#ifdef HAVE_EVP_PKEY_CTX_SET1_SCRYPT_SALT + BOOST_CHECK(CredentialsHolder::isHashingAvailable()); + const std::string sampleHash("$scrypt$ln=10,p=1,r=8$1GZ10YdmSGtTmKK9jTH85Q==$JHeICW1mUCnTC+nnULDr7QFQ3kRrZ7u12djruJdrPhI="); + + auto fromHashedHolder = CredentialsHolder(std::string(sampleHash), true); + BOOST_CHECK(fromHashedHolder.wasHashed()); + BOOST_CHECK(fromHashedHolder.isHashed()); + BOOST_CHECK(fromHashedHolder.matches(plaintext)); + BOOST_CHECK(!fromHashedHolder.matches("not test")); + + auto fromPlaintextHolder = CredentialsHolder(std::string(plaintext), true); + BOOST_CHECK(!fromPlaintextHolder.wasHashed()); + BOOST_CHECK(fromPlaintextHolder.isHashed()); + BOOST_CHECK(fromPlaintextHolder.matches(plaintext)); + BOOST_CHECK(!fromPlaintextHolder.matches("not test")); +#else + BOOST_CHECK(!CredentialsHolder::isHashingAvailable()); +#endif +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/test-dnsdist_cc.cc b/pdns/test-dnsdist_cc.cc index 2d49ad24398a..8bf0e99b26c5 100644 --- a/pdns/test-dnsdist_cc.cc +++ b/pdns/test-dnsdist_cc.cc @@ -488,7 +488,7 @@ BOOST_AUTO_TEST_CASE(replaceECSWithSameSize) { ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -523,7 +523,7 @@ BOOST_AUTO_TEST_CASE(replaceECSWithSameSizeAlreadyParsed) { ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -568,7 +568,7 @@ BOOST_AUTO_TEST_CASE(replaceECSWithSmaller) { ecsOpts.source = Netmask(origRemote, 32); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -606,7 +606,7 @@ BOOST_AUTO_TEST_CASE(replaceECSWithLarger) { ecsOpts.source = Netmask(origRemote, 8); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -659,7 +659,7 @@ BOOST_AUTO_TEST_CASE(replaceECSFollowedByTSIG) { ecsOpts.source = Netmask(origRemote, 8); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.addOpt(512, 0, 0, opts); pw.startRecord(DNSName("tsigname."), QType::TSIG, 0, QClass::ANY, DNSResourceRecord::ADDITIONAL, false); pw.commit(); @@ -715,7 +715,7 @@ BOOST_AUTO_TEST_CASE(replaceECSAfterAN) { ecsOpts.source = Netmask(origRemote, 8); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -770,7 +770,7 @@ BOOST_AUTO_TEST_CASE(replaceECSAfterAuth) { ecsOpts.source = Netmask(origRemote, 8); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -823,7 +823,7 @@ BOOST_AUTO_TEST_CASE(replaceECSBetweenTwoRecords) { ecsOpts.source = Netmask(origRemote, 8); string origECSOption = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOption)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOption); pw.startRecord(DNSName("additional"), QType::A, 0, QClass::IN, DNSResourceRecord::ADDITIONAL, false); pw.xfr32BitInt(0x01020304); pw.addOpt(512, 0, 0, opts); @@ -1074,7 +1074,7 @@ BOOST_AUTO_TEST_CASE(removeECSWhenOnlyOption) { ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1120,13 +1120,11 @@ BOOST_AUTO_TEST_CASE(removeECSWhenFirstOption) { EDNSSubnetOpts ecsOpts; ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV6); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr = cookiesOpt.makeOptString(); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1173,16 +1171,14 @@ BOOST_AUTO_TEST_CASE(removeECSWhenIntermediaryOption) { ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr1 = makeEDNSCookiesOptString(cookiesOpt); - string cookiesOptionStr2 = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr1 = cookiesOpt.makeOptString(); + string cookiesOptionStr2 = cookiesOpt.makeOptString(); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr1)); - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr2)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr1); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr2); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1225,16 +1221,14 @@ BOOST_AUTO_TEST_CASE(removeECSWhenLastOption) { pw.xfr32BitInt(0x01020304); pw.commit(); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr = cookiesOpt.makeOptString(); EDNSSubnetOpts ecsOpts; ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1277,7 +1271,7 @@ BOOST_AUTO_TEST_CASE(rewritingWithoutECSWhenOnlyOption) { ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1313,13 +1307,11 @@ BOOST_AUTO_TEST_CASE(rewritingWithoutECSWhenFirstOption) { EDNSSubnetOpts ecsOpts; ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr = cookiesOpt.makeOptString(); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1355,15 +1347,13 @@ BOOST_AUTO_TEST_CASE(rewritingWithoutECSWhenIntermediaryOption) { EDNSSubnetOpts ecsOpts; ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr1 = makeEDNSCookiesOptString(cookiesOpt); - string cookiesOptionStr2 = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr1 = cookiesOpt.makeOptString(); + string cookiesOptionStr2 = cookiesOpt.makeOptString(); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr1)); - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr2)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr1); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr2); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1399,13 +1389,11 @@ BOOST_AUTO_TEST_CASE(rewritingWithoutECSWhenLastOption) { EDNSSubnetOpts ecsOpts; ecsOpts.source = Netmask(origRemote, ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr = cookiesOpt.makeOptString(); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1467,13 +1455,11 @@ BOOST_AUTO_TEST_CASE(test_getEDNSZ) { EDNSSubnetOpts ecsOpts; ecsOpts.source = Netmask(ComboAddress("127.0.0.1"), ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr = cookiesOpt.makeOptString(); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); { /* no EDNS */ @@ -1565,13 +1551,11 @@ BOOST_AUTO_TEST_CASE(test_addEDNSToQueryTurnedResponse) { EDNSSubnetOpts ecsOpts; ecsOpts.source = Netmask(ComboAddress("127.0.0.1"), ECSSourcePrefixV4); string origECSOptionStr = makeEDNSSubnetOptsString(ecsOpts); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr = cookiesOpt.makeOptString(); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, origECSOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, origECSOptionStr); ComboAddress lc("127.0.0.1"); ComboAddress rem("127.0.0.1"); struct timespec queryRealTime; @@ -1672,7 +1656,7 @@ BOOST_AUTO_TEST_CASE(test_getEDNSOptionsStart) { ecsOpts.source = Netmask(ComboAddress("127.0.0.1"), ECSSourcePrefixV4); const string ecsOptionStr = makeEDNSSubnetOptsString(ecsOpts); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, ecsOptionStr)); + opts.emplace_back(EDNSOptionCode::ECS, ecsOptionStr); const ComboAddress lc("127.0.0.1"); const ComboAddress rem("127.0.0.1"); uint16_t optRDPosition; @@ -1774,16 +1758,14 @@ BOOST_AUTO_TEST_CASE(test_isEDNSOptionInOpt) { const string ecsOptionStr = makeEDNSSubnetOptsString(ecsOpts); const size_t sizeOfECSContent = ecsOptionStr.size(); const size_t sizeOfECSOption = /* option code */ 2 + /* option length */ 2 + sizeOfECSContent; - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - const string cookiesOptionStr = makeEDNSCookiesOptString(cookiesOpt); - const size_t sizeOfCookieOption = /* option code */ 2 + /* option length */ 2 + cookiesOpt.client.size() + cookiesOpt.server.size(); + EDNSCookiesOpt cookiesOpt("deadbeefdeadbeef"); + string cookiesOptionStr = cookiesOpt.makeOptString(); + const size_t sizeOfCookieOption = /* option code */ 2 + /* option length */ 2 + cookiesOpt.size(); /* GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, ecsOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, ecsOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); */ const ComboAddress lc("127.0.0.1"); const ComboAddress rem("127.0.0.1"); @@ -1829,8 +1811,8 @@ BOOST_AUTO_TEST_CASE(test_isEDNSOptionInOpt) { PacketBuffer query; GenericDNSPacketWriter pw(query, qname, qtype, qclass, 0); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1847,8 +1829,8 @@ BOOST_AUTO_TEST_CASE(test_isEDNSOptionInOpt) { PacketBuffer query; GenericDNSPacketWriter pw(query, qname, qtype, qclass, 0); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, ecsOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, ecsOptionStr)); + opts.emplace_back(EDNSOptionCode::ECS, ecsOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, ecsOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1869,9 +1851,9 @@ BOOST_AUTO_TEST_CASE(test_isEDNSOptionInOpt) { PacketBuffer query; GenericDNSPacketWriter pw(query, qname, qtype, qclass, 0); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::ECS, ecsOptionStr)); - opts.push_back(make_pair(EDNSOptionCode::COOKIE, cookiesOptionStr)); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); + opts.emplace_back(EDNSOptionCode::ECS, ecsOptionStr); + opts.emplace_back(EDNSOptionCode::COOKIE, cookiesOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -1892,8 +1874,8 @@ BOOST_AUTO_TEST_CASE(test_isEDNSOptionInOpt) { PacketBuffer query; GenericDNSPacketWriter pw(query, qname, qtype, qclass, 0); GenericDNSPacketWriter::optvect_t opts; - opts.push_back(make_pair(EDNSOptionCode::ECS, ecsOptionStr)); - opts.push_back(make_pair(65535, cookiesOptionStr)); + opts.emplace_back(EDNSOptionCode::ECS, ecsOptionStr); + opts.emplace_back(65535, cookiesOptionStr); pw.addOpt(512, 0, 0, opts); pw.commit(); diff --git a/pdns/test-dnsdistpacketcache_cc.cc b/pdns/test-dnsdistpacketcache_cc.cc index 7af31fa110aa..4be01c2fcf1a 100644 --- a/pdns/test-dnsdistpacketcache_cc.cc +++ b/pdns/test-dnsdistpacketcache_cc.cc @@ -569,7 +569,7 @@ BOOST_AUTO_TEST_CASE(test_PCCollision) { GenericDNSPacketWriter::optvect_t ednsOptions; EDNSSubnetOpts opt; opt.source = Netmask("10.0.59.220/32"); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pwQ.addOpt(512, 0, 0, ednsOptions); pwQ.commit(); @@ -612,7 +612,7 @@ BOOST_AUTO_TEST_CASE(test_PCCollision) { GenericDNSPacketWriter::optvect_t ednsOptions; EDNSSubnetOpts opt; opt.source = Netmask("10.0.167.48/32"); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pwQ.addOpt(512, 0, 0, ednsOptions); pwQ.commit(); @@ -649,11 +649,11 @@ BOOST_AUTO_TEST_CASE(test_PCCollision) { pwFQ.getHeader()->id = 0x42; opt.source = Netmask("10." + std::to_string(idxA) + "." + std::to_string(idxB) + "." + std::to_string(idxC) + "/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pwFQ.addOpt(512, 0, 0, ednsOptions); pwFQ.commit(); secondKey = pc.getKey(qname.toDNSString(), qname.wirelength(), secondQuery, false); - auto pair = colMap.insert(std::make_pair(secondKey, opt.source)); + auto pair = colMap.emplace(secondKey, opt.source); total++; if (!pair.second) { collisions++; diff --git a/pdns/test-dnsname_cc.cc b/pdns/test-dnsname_cc.cc index af1b2571c169..98aed701bf73 100644 --- a/pdns/test-dnsname_cc.cc +++ b/pdns/test-dnsname_cc.cc @@ -388,35 +388,55 @@ BOOST_AUTO_TEST_CASE(test_hashContainer) { BOOST_AUTO_TEST_CASE(test_QuestionHash) { - vector packet; + vector packet(sizeof(dnsheader)); reportBasicTypes(); - DNSPacketWriter dpw1(packet, DNSName("www.ds9a.nl."), QType::AAAA); - - auto hash1=hashQuestion((char*)&packet[0], packet.size(), 0); - DNSPacketWriter dpw2(packet, DNSName("wWw.Ds9A.nL."), QType::AAAA); - auto hash2=hashQuestion((char*)&packet[0], packet.size(), 0); + + // A return init case + BOOST_CHECK_EQUAL(hashQuestion(packet.data(), sizeof(dnsheader), 0xffU), 0xffU); + + // We subtract 4 from the packet sizes since DNSPacketWriter adds a type and a class + // We expect the hash of the root to be unequal to the burtle init value + DNSPacketWriter dpw0(packet, DNSName("."), QType::AAAA); + BOOST_CHECK(hashQuestion(packet.data(), packet.size() - 4, 0xffU) != 0xffU); + + // A truncated buffer should return the init value + DNSPacketWriter dpw1(packet, DNSName("."), QType::AAAA); + BOOST_CHECK_EQUAL(hashQuestion(packet.data(), packet.size() - 5, 0xffU), 0xffU); + + DNSPacketWriter dpw2(packet, DNSName("www.ds9a.nl."), QType::AAAA); + // Let's make an invalid name by overwriting the length of the second label just outside the buffer + packet[sizeof(dnsheader) + 4] = 8; + BOOST_CHECK_EQUAL(hashQuestion(packet.data(), packet.size() - 4, 0xffU), 0xffU); + + DNSPacketWriter dpw3(packet, DNSName("www.ds9a.nl."), QType::AAAA); + // Let's make an invalid name by overwriting the length of the second label way outside the buffer + packet[sizeof(dnsheader) + 4] = 0xff; + BOOST_CHECK_EQUAL(hashQuestion(packet.data(), packet.size() - 4, 0xffU), 0xffU); + + DNSPacketWriter dpw4(packet, DNSName("www.ds9a.nl."), QType::AAAA); + auto hash1 = hashQuestion(&packet[0], packet.size() - 4, 0); + DNSPacketWriter dpw5(packet, DNSName("wWw.Ds9A.nL."), QType::AAAA); + auto hash2 = hashQuestion(&packet[0], packet.size() - 4, 0); BOOST_CHECK_EQUAL(hash1, hash2); - + vector counts(1500); - - for(unsigned int n=0; n < 100000; ++n) { + for(unsigned int n = 0; n < 100000; ++n) { packet.clear(); - DNSPacketWriter dpw3(packet, DNSName(std::to_string(n)+"."+std::to_string(n*2)+"."), QType::AAAA); - counts[hashQuestion((char*)&packet[0], packet.size(), 0) % counts.size()]++; + DNSPacketWriter dpw(packet, DNSName(std::to_string(n) + "." + std::to_string(n*2) + "."), QType::AAAA); + counts[hashQuestion(&packet[0], packet.size() - 4, 0) % counts.size()]++; } - + double sum = std::accumulate(std::begin(counts), std::end(counts), 0.0); double m = sum / counts.size(); - + double accum = 0.0; std::for_each (std::begin(counts), std::end(counts), [&](const double d) { accum += (d - m) * (d - m); }); - + double stdev = sqrt(accum / (counts.size()-1)); - BOOST_CHECK(stdev < 10); + BOOST_CHECK(stdev < 10); } - BOOST_AUTO_TEST_CASE(test_packetParse) { vector packet; diff --git a/pdns/test-dnsrecords_cc.cc b/pdns/test-dnsrecords_cc.cc index 6c66cb2312c4..9079c6ed5779 100644 --- a/pdns/test-dnsrecords_cc.cc +++ b/pdns/test-dnsrecords_cc.cc @@ -444,7 +444,7 @@ BOOST_AUTO_TEST_CASE(test_opt_record_out) { DNSPacketWriter pw(pak, DNSName("www.powerdns.com"), QType::A); pw.startRecord(DNSName("www.powerdns.com"), QType::A, 16, 1, DNSResourceRecord::ANSWER); pw.xfrIP(htonl(0x7f000001)); - opts.push_back(pair(3, "powerdns")); + opts.emplace_back(3, "powerdns"); pw.addOpt(1280, 0, 0, opts); pw.getHeader()->id = htons(0xf001); pw.getHeader()->rd = 1; diff --git a/pdns/test-ednscookie_cc.cc b/pdns/test-ednscookie_cc.cc new file mode 100644 index 000000000000..556429fcfb43 --- /dev/null +++ b/pdns/test-ednscookie_cc.cc @@ -0,0 +1,108 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_NO_MAIN + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "ednscookies.hh" + +#include + +BOOST_AUTO_TEST_SUITE(test_ednscookie) +BOOST_AUTO_TEST_CASE(test_getEDNSCookiesOptFromString) +{ + string cookie(""); + EDNSCookiesOpt eco(cookie); + // Length 0 + BOOST_CHECK(!eco.isWellFormed()); + + // Too short + cookie = "\x12\x34\x56\x78\x90\xab\xcd"; + BOOST_CHECK(!eco.makeFromString(cookie)); + + // Correct length client cookie + cookie = "\x12\x34\x56\x78\x90\xab\xcd\xef"; + BOOST_CHECK(eco.makeFromString(cookie)); + + // Too short server cookie + cookie = "\x12\x34\x56\x78\x90\xab\xcd\xef\x01"; + BOOST_CHECK(!eco.makeFromString(cookie)); + + cookie = "\x12\x34\x56\x78\x90\xab\xcd\xef\x12\x34\x56\x78\x90\xab\xcd"; + BOOST_CHECK(!eco.makeFromString(cookie)); + + // Have server cookie of correct length + cookie = "\x12\x34\x56\x78\x90\xab\xcd\xef"; + cookie += cookie; // size 16 + BOOST_CHECK(eco.makeFromString(cookie)); + + cookie += cookie; // size 32 + BOOST_CHECK(eco.makeFromString(cookie)); + + cookie += "\x12\x34\x56\x78\x90\xab\xcd\xef"; // size 40 (the max) + BOOST_CHECK(eco.makeFromString(cookie)); + + // Cookie total size too long + cookie += "\x01"; + BOOST_CHECK(!eco.makeFromString(cookie)); +} + +BOOST_AUTO_TEST_CASE(test_ctor) +{ + string cookie(""); + auto eco = EDNSCookiesOpt(cookie); + BOOST_CHECK(!eco.isWellFormed()); + + eco = EDNSCookiesOpt("\x12\x34\x56\x78\x90\xab\xcd\xef"); + BOOST_CHECK(eco.isWellFormed()); + BOOST_CHECK_EQUAL(8U, eco.makeOptString().length()); +} + +#ifdef HAVE_CRYPTO_SHORTHASH +BOOST_AUTO_TEST_CASE(test_createEDNSServerCookie) +{ + auto eco = EDNSCookiesOpt("\x12\x34\x56\x78\x90\xab\xcd\xef"); + ComboAddress remote("192.0.2.2"); + + BOOST_CHECK(eco.isWellFormed()); + + // wrong keysize (not 128 bits) + string secret = "blablablabla"; + BOOST_CHECK(!eco.makeServerCookie(secret, remote)); + BOOST_CHECK(eco.isWellFormed()); + BOOST_CHECK(!eco.isValid(secret, remote)); + + secret = "blablablablablab"; + BOOST_CHECK(eco.makeServerCookie(secret, remote)); + BOOST_CHECK(eco.isWellFormed()); + BOOST_CHECK(eco.isValid(secret, remote)); + + EDNSCookiesOpt eco2(eco.makeOptString()); + BOOST_CHECK(!eco2.isValid(secret, ComboAddress("192.0.2.1"))); + BOOST_CHECK(!eco2.isValid("blablablablabla1", remote)); + BOOST_CHECK(eco2.isValid(secret, remote)); +} +#endif + +BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/test-misc_hh.cc b/pdns/test-misc_hh.cc index bc10168df2fd..248ef8d61168 100644 --- a/pdns/test-misc_hh.cc +++ b/pdns/test-misc_hh.cc @@ -40,15 +40,15 @@ BOOST_AUTO_TEST_CASE(test_CIStringCompare) { BOOST_AUTO_TEST_CASE(test_CIStringPairCompare) { set nsset2; - nsset2.insert(make_pair("ns.example.com", 1)); - nsset2.insert(make_pair("abc", 1)); - nsset2.insert(make_pair("", 1)); - nsset2.insert(make_pair("def", 1)); - nsset2.insert(make_pair("abc", 2)); - nsset2.insert(make_pair("abc", 1)); - nsset2.insert(make_pair("ns.example.com", 0)); - nsset2.insert(make_pair("abc", 2)); - nsset2.insert(make_pair("ABC", 2)); + nsset2.emplace("ns.example.com", 1); + nsset2.emplace("abc", 1); + nsset2.emplace("", 1); + nsset2.emplace("def", 1); + nsset2.emplace("abc", 2); + nsset2.emplace("abc", 1); + nsset2.emplace("ns.example.com", 0); + nsset2.emplace("abc", 2); + nsset2.emplace("ABC", 2); BOOST_CHECK_EQUAL(nsset2.size(), 6U); ostringstream s; @@ -391,4 +391,11 @@ BOOST_AUTO_TEST_CASE(test_parseSVCBValueList) BOOST_CHECK_EQUAL(out[0], "foobar123 blabla bla,baz quux456"); } +BOOST_AUTO_TEST_CASE(test_makeBytesFromHex) { + string out = makeBytesFromHex("1234567890abcdef"); + BOOST_CHECK_EQUAL(out, "\x12\x34\x56\x78\x90\xab\xcd\xef"); + + BOOST_CHECK_THROW(makeBytesFromHex("123"), std::range_error); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/pdns/test-nameserver_cc.cc b/pdns/test-nameserver_cc.cc index 2f6044dbbb22..0517b076f67b 100644 --- a/pdns/test-nameserver_cc.cc +++ b/pdns/test-nameserver_cc.cc @@ -11,6 +11,8 @@ #include extern vector g_localaddresses; +NetmaskGroup g_proxyProtocolACL; +size_t g_proxyProtocolMaximumSize = 512; BOOST_AUTO_TEST_SUITE(test_nameserver_cc) diff --git a/pdns/test-packetcache_cc.cc b/pdns/test-packetcache_cc.cc index e37c4b062ee3..4b3a39936a1e 100644 --- a/pdns/test-packetcache_cc.cc +++ b/pdns/test-packetcache_cc.cc @@ -165,7 +165,7 @@ try we directly compute the hash instead of querying the cache because 1/ it's faster 2/ no deferred-lookup issues */ - q.setHash(g_PC->canHashPacket(q.getString(), false)); + q.setHash(g_PC->canHashPacket(q.getString())); const unsigned int maxTTL = 3600; g_PC->insert(q, r, maxTTL); @@ -347,7 +347,7 @@ BOOST_AUTO_TEST_CASE(test_AuthPacketCache) { { ecsOpts.source = Netmask(ComboAddress("192.0.2.1"), 32); - opts.push_back(make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(ecsOpts))); + opts.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(ecsOpts)); DNSPacketWriter pw(pak, DNSName("www.powerdns.com"), QType::A); pw.addOpt(512, 0, 0, opts); pw.commit(); @@ -359,7 +359,7 @@ BOOST_AUTO_TEST_CASE(test_AuthPacketCache) { { DNSPacketWriter pw(pak, DNSName("www.powerdns.com"), QType::A); ecsOpts.source = Netmask(ComboAddress("192.0.2.2"), 32); - opts.push_back(make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(ecsOpts))); + opts.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(ecsOpts)); pw.addOpt(512, 0, 0, opts); pw.commit(); ecs2.parse((char*)&pak[0], pak.size()); @@ -370,7 +370,7 @@ BOOST_AUTO_TEST_CASE(test_AuthPacketCache) { { DNSPacketWriter pw(pak, DNSName("www.powerdns.com"), QType::A); ecsOpts.source = Netmask(ComboAddress("192.0.2.3"), 16); - opts.push_back(make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(ecsOpts))); + opts.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(ecsOpts)); pw.addOpt(512, 0, 0, opts); pw.commit(); ecs3.parse((char*)&pak[0], pak.size()); diff --git a/pdns/test-packetcache_hh.cc b/pdns/test-packetcache_hh.cc index 6a55effcffe8..1e361a7fa66e 100644 --- a/pdns/test-packetcache_hh.cc +++ b/pdns/test-packetcache_hh.cc @@ -32,7 +32,7 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw1.getHeader()->qr = false; pw1.getHeader()->id = 0x42; string spacket1((const char*)&packet[0], packet.size()); - auto hash1 = PacketCache::canHashPacket(spacket1, false); + auto hash1 = PacketCache::canHashPacket(spacket1, optionsToSkip); packet.clear(); DNSPacketWriter pw2(packet, qname, qtype); @@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw2.getHeader()->qr = false; pw2.getHeader()->id = 0x84; string spacket2((const char*)&packet[0], packet.size()); - auto hash2 = PacketCache::canHashPacket(spacket2, false); + auto hash2 = PacketCache::canHashPacket(spacket2, optionsToSkip); BOOST_CHECK_EQUAL(hash1, hash2); BOOST_CHECK(PacketCache::queryMatches(spacket1, spacket2, qname, optionsToSkip)); @@ -55,12 +55,12 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw1.getHeader()->id = 0x42; opt.source = Netmask("10.0.152.74/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pw1.addOpt(512, 0, 0, ednsOptions); pw1.commit(); string spacket1((const char*)&packet[0], packet.size()); - auto hash1 = PacketCache::canHashPacket(spacket1, false); + auto hash1 = PacketCache::canHashPacket(spacket1, optionsToSkip); packet.clear(); DNSPacketWriter pw2(packet, qname, qtype); @@ -69,12 +69,12 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw2.getHeader()->id = 0x84; opt.source = Netmask("10.2.70.250/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pw2.addOpt(512, 0, 0, ednsOptions); pw2.commit(); string spacket2((const char*)&packet[0], packet.size()); - auto hash2 = PacketCache::canHashPacket(spacket2, false); + auto hash2 = PacketCache::canHashPacket(spacket2, optionsToSkip); BOOST_CHECK_EQUAL(hash1, hash2); /* the hash is the same but we should _not_ match */ @@ -97,11 +97,11 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pwFQ.getHeader()->id = 0x42; opt.source = Netmask("10." + std::to_string(idxA) + "." + std::to_string(idxB) + "." + std::to_string(idxC) + "/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pwFQ.addOpt(512, 0, 0, ednsOptions); pwFQ.commit(); - auto secondKey = PacketCache::canHashPacket(std::string(reinterpret_cast(secondQuery.data()), secondQuery.size()), false); - auto pair = colMap.insert(std::make_pair(secondKey, opt.source)); + auto secondKey = PacketCache::canHashPacket(std::string(reinterpret_cast(secondQuery.data()), secondQuery.size()), optionsToSkip); + auto pair = colMap.emplace(secondKey, opt.source); total++; if (!pair.second) { collisions++; @@ -127,12 +127,12 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw1.getHeader()->id = 0x42; opt.source = Netmask("10.0.34.159/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pw1.addOpt(512, 0, EDNSOpts::DNSSECOK, ednsOptions); pw1.commit(); string spacket1((const char*)&packet[0], packet.size()); - auto hash1 = PacketCache::canHashPacket(spacket1, false); + auto hash1 = PacketCache::canHashPacket(spacket1, optionsToSkip); packet.clear(); DNSPacketWriter pw2(packet, qname, qtype); @@ -141,13 +141,13 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw2.getHeader()->id = 0x84; opt.source = Netmask("10.0.179.58/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); /* no EDNSOpts::DNSSECOK !! */ pw2.addOpt(512, 0, 0, ednsOptions); pw2.commit(); string spacket2((const char*)&packet[0], packet.size()); - auto hash2 = PacketCache::canHashPacket(spacket2, false); + auto hash2 = PacketCache::canHashPacket(spacket2, optionsToSkip); BOOST_CHECK_EQUAL(hash1, hash2); /* the hash is the same but we should _not_ match */ @@ -163,16 +163,14 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw1.getHeader()->id = 0x42; opt.source = Netmask("192.0.2.1/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::COOKIE, makeEDNSCookiesOptString(cookiesOpt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); + EDNSCookiesOpt cookiesOpt(string("deadbeefdeadbeef")); + ednsOptions.emplace_back(EDNSOptionCode::COOKIE, cookiesOpt.makeOptString()); pw1.addOpt(512, 0, EDNSOpts::DNSSECOK, ednsOptions); pw1.commit(); string spacket1((const char*)&packet[0], packet.size()); - auto hash1 = PacketCache::canHashPacket(spacket1, false); + auto hash1 = PacketCache::canHashPacket(spacket1, optionsToSkip); packet.clear(); DNSPacketWriter pw2(packet, qname, qtype); @@ -181,15 +179,14 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pw2.getHeader()->id = 0x84; opt.source = Netmask("192.0.2.1/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("badc0fee"); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::COOKIE, makeEDNSCookiesOptString(cookiesOpt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); + cookiesOpt.makeFromString(string("deadbeefbadc0fee")); + ednsOptions.emplace_back(EDNSOptionCode::COOKIE, cookiesOpt.makeOptString()); pw2.addOpt(512, 0, EDNSOpts::DNSSECOK, ednsOptions); pw2.commit(); string spacket2((const char*)&packet[0], packet.size()); - auto hash2 = PacketCache::canHashPacket(spacket2, false); + auto hash2 = PacketCache::canHashPacket(spacket2, optionsToSkip); BOOST_CHECK_EQUAL(hash1, hash2); /* the hash is the same but we should _not_ match */ @@ -214,11 +211,11 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pwFQ.getHeader()->id = 0x42; opt.source = Netmask("10." + std::to_string(idxA) + "." + std::to_string(idxB) + "." + std::to_string(idxC) + "/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pwFQ.addOpt(512, 0, 32768, ednsOptions); pwFQ.commit(); - auto secondKey = PacketCache::canHashPacket(std::string(reinterpret_cast(secondQuery.data()), secondQuery.size()), false); - colMap.insert(std::make_pair(secondKey, opt.source)); + auto secondKey = PacketCache::canHashPacket(std::string(reinterpret_cast(secondQuery.data()), secondQuery.size()), optionsToSkip); + colMap.emplace(secondKey, opt.source); secondQuery.clear(); DNSPacketWriter pwSQ(secondQuery, qname, QType::AAAA, QClass::IN, 0); @@ -227,10 +224,10 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheAuthCollision) { pwSQ.getHeader()->id = 0x42; opt.source = Netmask("10." + std::to_string(idxA) + "." + std::to_string(idxB) + "." + std::to_string(idxC) + "/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pwSQ.addOpt(512, 0, 0, ednsOptions); pwSQ.commit(); - secondKey = PacketCache::canHashPacket(std::string(reinterpret_cast(secondQuery.data()), secondQuery.size()), false); + secondKey = PacketCache::canHashPacket(std::string(reinterpret_cast(secondQuery.data()), secondQuery.size()), optionsToSkip); total++; if (colMap.count(secondKey)) { @@ -255,6 +252,7 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecSimple) { uint16_t qtype = QType::AAAA; EDNSSubnetOpts opt; DNSPacketWriter::optvect_t ednsOptions; + static const std::unordered_set optionsToSkip{ EDNSOptionCode::COOKIE, EDNSOptionCode::ECS }; { vector packet; @@ -272,7 +270,7 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecSimple) { *(ptr + 1) = 255; /* truncate the end of the OPT header to try to trigger an out of bounds read */ spacket1.resize(spacket1.size() - 6); - BOOST_CHECK_NO_THROW(PacketCache::canHashPacket(spacket1, true)); + BOOST_CHECK_NO_THROW(PacketCache::canHashPacket(spacket1, optionsToSkip)); } } @@ -293,7 +291,7 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecCollision) { pw1.getHeader()->qr = false; pw1.getHeader()->id = 0x42; string spacket1((const char*)&packet[0], packet.size()); - auto hash1 = PacketCache::canHashPacket(spacket1, true); + auto hash1 = PacketCache::canHashPacket(spacket1, optionsToSkip); packet.clear(); DNSPacketWriter pw2(packet, qname, qtype); @@ -301,7 +299,7 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecCollision) { pw2.getHeader()->qr = false; pw2.getHeader()->id = 0x84; string spacket2((const char*)&packet[0], packet.size()); - auto hash2 = PacketCache::canHashPacket(spacket2, true); + auto hash2 = PacketCache::canHashPacket(spacket2, optionsToSkip); BOOST_CHECK_EQUAL(hash1, hash2); BOOST_CHECK(PacketCache::queryMatches(spacket1, spacket2, qname, optionsToSkip)); @@ -316,12 +314,12 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecCollision) { pw1.getHeader()->id = 0x42; opt.source = Netmask("10.0.18.199/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pw1.addOpt(512, 0, 0, ednsOptions); pw1.commit(); string spacket1((const char*)&packet[0], packet.size()); - auto hash1 = PacketCache::canHashPacket(spacket1, true); + auto hash1 = PacketCache::canHashPacket(spacket1, optionsToSkip); packet.clear(); DNSPacketWriter pw2(packet, qname, qtype); @@ -330,12 +328,12 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecCollision) { pw2.getHeader()->id = 0x84; opt.source = Netmask("10.0.131.66/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); pw2.addOpt(512, 0, 0, ednsOptions); pw2.commit(); string spacket2((const char*)&packet[0], packet.size()); - auto hash2 = PacketCache::canHashPacket(spacket2, true); + auto hash2 = PacketCache::canHashPacket(spacket2, optionsToSkip); BOOST_CHECK_EQUAL(hash1, hash2); /* the hash is the same and we don't hash the ECS so we should match */ @@ -351,20 +349,14 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecCollision) { pw1.getHeader()->id = 0x42; opt.source = Netmask("192.0.2.1/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); - EDNSCookiesOpt cookiesOpt; - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - cookiesOpt.server[4] = -20; - cookiesOpt.server[5] = -114; - cookiesOpt.server[6] = 0; - cookiesOpt.server[7] = 0; - ednsOptions.push_back(std::make_pair(EDNSOptionCode::COOKIE, makeEDNSCookiesOptString(cookiesOpt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); + EDNSCookiesOpt cookiesOpt(string("deadbeefdead\x11\xee\x00\x00").c_str(), 16); + ednsOptions.emplace_back(EDNSOptionCode::COOKIE, cookiesOpt.makeOptString()); pw1.addOpt(512, 0, EDNSOpts::DNSSECOK, ednsOptions); pw1.commit(); string spacket1((const char*)&packet[0], packet.size()); - auto hash1 = PacketCache::canHashPacket(spacket1, true); + auto hash1 = PacketCache::canHashPacket(spacket1, optionsToSkip); packet.clear(); DNSPacketWriter pw2(packet, qname, qtype); @@ -373,19 +365,14 @@ BOOST_AUTO_TEST_CASE(test_PacketCacheRecCollision) { pw2.getHeader()->id = 0x84; opt.source = Netmask("192.0.2.1/32"); ednsOptions.clear(); - ednsOptions.push_back(std::make_pair(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt))); - cookiesOpt.client = string("deadbeef"); - cookiesOpt.server = string("deadbeef"); - cookiesOpt.server[4] = 103; - cookiesOpt.server[5] = 68; - cookiesOpt.server[6] = 0; - cookiesOpt.server[7] = 0; - ednsOptions.push_back(std::make_pair(EDNSOptionCode::COOKIE, makeEDNSCookiesOptString(cookiesOpt))); + ednsOptions.emplace_back(EDNSOptionCode::ECS, makeEDNSSubnetOptsString(opt)); + cookiesOpt.makeFromString(string("deadbeefdead\x67\x44\x00\x00").c_str(), 16); + ednsOptions.emplace_back(EDNSOptionCode::COOKIE, cookiesOpt.makeOptString()); pw2.addOpt(512, 0, EDNSOpts::DNSSECOK, ednsOptions); pw2.commit(); string spacket2((const char*)&packet[0], packet.size()); - auto hash2 = PacketCache::canHashPacket(spacket2, true); + auto hash2 = PacketCache::canHashPacket(spacket2, optionsToSkip); BOOST_CHECK_EQUAL(hash1, hash2); /* the hash is the same but we should _not_ match, even though we skip the ECS part, because the cookies are different */ diff --git a/pdns/test-rcpgenerator_cc.cc b/pdns/test-rcpgenerator_cc.cc index c2ea8ea4f4f0..0987108e2287 100644 --- a/pdns/test-rcpgenerator_cc.cc +++ b/pdns/test-rcpgenerator_cc.cc @@ -67,6 +67,28 @@ BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_alpn) { RecordTextWriter rtw2(target); rtw2.xfrSvcParamKeyVals(v); BOOST_CHECK_EQUAL(target, source); + + // Check generic + v.clear(); + source="key1=\\002h2\\002h3"; + RecordTextReader rtr3(source); + rtr3.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + alpn = v.begin()->getALPN(); + BOOST_CHECK_EQUAL(alpn.size(), 2U); + val = alpn.begin(); + BOOST_CHECK_EQUAL(*val, "h2"); + val++; + BOOST_CHECK_EQUAL(*val, "h3"); + + // Error conditions + source="key1=\\002h2\\003h3"; // Wrong length for 2nd argument + RecordTextReader rtr4(source); + BOOST_CHECK_THROW(rtr4.xfrSvcParamKeyVals(v), RecordTextException); + + source="key1=\\002h2\\002h3foobar"; // extra data + RecordTextReader rtr5(source); + BOOST_CHECK_THROW(rtr5.xfrSvcParamKeyVals(v), RecordTextException); } BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_mandatory) { @@ -103,6 +125,35 @@ BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_mandatory) { RecordTextWriter rtw2(target); rtw2.xfrSvcParamKeyVals(v); BOOST_CHECK_EQUAL(target, source); + + // Generic parsing + v.clear(); + source = "key0=\\000\\001\\000\\004"; + RecordTextReader rtr3(source); + rtr3.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + m = v.begin()->getMandatory(); + BOOST_CHECK_EQUAL(m.size(), 2U); + val = m.begin(); + BOOST_CHECK(*val == SvcParam::alpn); + val++; + BOOST_CHECK(*val == SvcParam::ipv4hint); + + // Broken + v.clear(); + source = "key0=\\000\\001\\000"; + RecordTextReader rtr4(source); + BOOST_CHECK_THROW(rtr4.xfrSvcParamKeyVals(v), RecordTextException); + + v.clear(); + source = "key0="; + RecordTextReader rtr5(source); + BOOST_CHECK_THROW(rtr5.xfrSvcParamKeyVals(v), RecordTextException); + + v.clear(); + source = "key0"; + RecordTextReader rtr6(source); + BOOST_CHECK_THROW(rtr6.xfrSvcParamKeyVals(v), RecordTextException); } BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_no_default_alpn) { @@ -123,6 +174,15 @@ BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_no_default_alpn) { RecordTextReader rtr2("no-default-alpn="); v.clear(); BOOST_CHECK_THROW(rtr2.xfrSvcParamKeyVals(v), RecordTextException); + + // Generic + v.clear(); + source = "key2"; + RecordTextReader rtr3(source); + rtr3.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + k = v.begin()->getKey(); + BOOST_CHECK(k == SvcParam::no_default_alpn); } BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_ipv4hint) { @@ -173,6 +233,33 @@ BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_ipv4hint) { v.clear(); RecordTextReader rtr4("ipv4hint=192.0.2.1,2001:db8::1"); BOOST_CHECK_THROW(rtr4.xfrSvcParamKeyVals(v), RecordTextException); + + // Check if we can parse the generic format + v.clear(); + source = "key4=\\192\\000\\002\\015"; + RecordTextReader rtr5(source); + rtr5.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().size(), 1U); + k = v.begin()->getKey(); + BOOST_CHECK(k == SvcParam::ipv4hint); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().begin()->toString(), "192.0.2.15"); + + v.clear(); + source = "key4=\\192\\000\\002\\015\\192\\000\\002\\222"; + RecordTextReader rtr6(source); + rtr6.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().size(), 2U); + k = v.begin()->getKey(); + BOOST_CHECK(k == SvcParam::ipv4hint); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().begin()->toString(), "192.0.2.15"); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().at(1).toString(), "192.0.2.222"); + + v.clear(); + source = "key4=\\192\\000\\222"; // Wrong number of octets + RecordTextReader rtr7(source); + BOOST_CHECK_THROW(rtr7.xfrSvcParamKeyVals(v), RecordTextException); } BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_ipv6hint) { @@ -223,6 +310,32 @@ BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_ipv6hint) { v.clear(); RecordTextReader rtr4("ipv6hint=192.0.2.1,2001:db8::1"); BOOST_CHECK_THROW(rtr4.xfrSvcParamKeyVals(v), RecordTextException); + + // Check if we can parse the generic format + v.clear(); + RecordTextReader rtr5("key6=\\032\\001\\013\\184\\000\\083\\000\\000\\000\\000\\000\\000\\000\\000\\000\\021"); + rtr5.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().size(), 1U); + k = v.begin()->getKey(); + BOOST_CHECK(k == SvcParam::ipv6hint); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().begin()->toString(), "2001:db8:53::15"); + + v.clear(); + source = "key6=\\032\\001\\013\\184\\000\\083\\000\\000\\000\\000\\000\\000\\000\\000\\000\\021\\032\\001\\013\\184\\000\\083\\000\\000\\000\\000\\000\\000\\000\\000\\000\\022"; + RecordTextReader rtr6(source); + rtr6.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().size(), 2U); + k = v.begin()->getKey(); + BOOST_CHECK(k == SvcParam::ipv6hint); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().begin()->toString(), "2001:db8:53::15"); + BOOST_CHECK_EQUAL(v.begin()->getIPHints().at(1).toString(), "2001:db8:53::16"); + + v.clear(); + source = "key6=\\040\\001\\015\\270\\000\\123\\000\\000\\000\\000\\000\\000\\000\\000\\000"; // wrong number of octets + RecordTextReader rtr7(source); + BOOST_CHECK_THROW(rtr7.xfrSvcParamKeyVals(v), RecordTextException); } BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_port) { @@ -257,6 +370,20 @@ BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_port) { v.clear(); RecordTextReader rtr5("port"); BOOST_CHECK_THROW(rtr5.xfrSvcParamKeyVals(v), RecordTextException); + + // Generic + v.clear(); + RecordTextReader rtr6("key3"); + BOOST_CHECK_THROW(rtr6.xfrSvcParamKeyVals(v), RecordTextException); + + v.clear(); + RecordTextReader rtr7("key3=\\000\\053"); + rtr7.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + k = v.begin()->getKey(); + BOOST_CHECK(k == SvcParam::port); + val = v.begin()->getPort(); + BOOST_CHECK_EQUAL(val, 53); } BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_generic) { @@ -378,6 +505,17 @@ BOOST_AUTO_TEST_CASE(test_xfrSvcParamKeyVals_ech) { RecordTextWriter rtw(target); rtw.xfrSvcParamKeyVals(v); BOOST_CHECK_EQUAL(source, target); + + // Generic + v.clear(); + source = "key5=echconfig"; + RecordTextReader rtr2(source); + rtr2.xfrSvcParamKeyVals(v); + BOOST_CHECK_EQUAL(v.size(), 1U); + k = v.begin()->getKey(); + BOOST_CHECK(k == SvcParam::ech); + val = v.begin()->getECH(); + BOOST_CHECK_EQUAL(val, "echconfig"); } BOOST_AUTO_TEST_CASE(test_xfrNodeOrLocatorID) { diff --git a/pdns/test-recpacketcache_cc.cc b/pdns/test-recpacketcache_cc.cc index a70e0a874ba0..b79bf3c09113 100644 --- a/pdns/test-recpacketcache_cc.cc +++ b/pdns/test-recpacketcache_cc.cc @@ -13,27 +13,27 @@ #include "recpacketcache.hh" #include - BOOST_AUTO_TEST_SUITE(test_recpacketcache_cc) -BOOST_AUTO_TEST_CASE(test_recPacketCacheSimple) { +BOOST_AUTO_TEST_CASE(test_recPacketCacheSimple) +{ RecursorPacketCache rpc; string fpacket; - unsigned int tag=0; - uint32_t age=0; - uint32_t qhash=0; - uint32_t ttd=3600; + unsigned int tag = 0; + uint32_t age = 0; + uint32_t qhash = 0; + uint32_t ttd = 3600; BOOST_CHECK_EQUAL(rpc.size(), 0U); - ::arg().set("rng")="auto"; - ::arg().set("entropy-source")="/dev/urandom"; + ::arg().set("rng") = "auto"; + ::arg().set("entropy-source") = "/dev/urandom"; DNSName qname("www.powerdns.com"); vector packet; DNSPacketWriter pw(packet, qname, QType::A); - pw.getHeader()->rd=true; - pw.getHeader()->qr=false; - pw.getHeader()->id=dns_random_uint16(); + pw.getHeader()->rd = true; + pw.getHeader()->qr = false; + pw.getHeader()->id = dns_random_uint16(); string qpacket((const char*)&packet[0], packet.size()); pw.startRecord(qname, QType::A, ttd); @@ -67,12 +67,12 @@ BOOST_AUTO_TEST_CASE(test_recPacketCacheSimple) { BOOST_CHECK_EQUAL(fpacket, rpacket); packet.clear(); - qname+=DNSName("co.uk"); + qname += DNSName("co.uk"); DNSPacketWriter pw2(packet, qname, QType::A); - pw2.getHeader()->rd=true; - pw2.getHeader()->qr=false; - pw2.getHeader()->id=dns_random_uint16(); + pw2.getHeader()->rd = true; + pw2.getHeader()->qr = false; + pw2.getHeader()->id = dns_random_uint16(); qpacket.assign((const char*)&packet[0], packet.size()); found = rpc.getResponsePacket(tag, qpacket, time(nullptr), &fpacket, &age, &qhash); @@ -84,24 +84,25 @@ BOOST_AUTO_TEST_CASE(test_recPacketCacheSimple) { BOOST_CHECK_EQUAL(rpc.size(), 0U); } -BOOST_AUTO_TEST_CASE(test_recPacketCacheSimplePost2038) { +BOOST_AUTO_TEST_CASE(test_recPacketCacheSimplePost2038) +{ RecursorPacketCache rpc; string fpacket; - unsigned int tag=0; - uint32_t age=0; - uint32_t qhash=0; - uint32_t ttd=3600; + unsigned int tag = 0; + uint32_t age = 0; + uint32_t qhash = 0; + uint32_t ttd = 3600; BOOST_CHECK_EQUAL(rpc.size(), 0U); - ::arg().set("rng")="auto"; - ::arg().set("entropy-source")="/dev/urandom"; + ::arg().set("rng") = "auto"; + ::arg().set("entropy-source") = "/dev/urandom"; DNSName qname("www.powerdns.com"); vector packet; DNSPacketWriter pw(packet, qname, QType::A); - pw.getHeader()->rd=true; - pw.getHeader()->qr=false; - pw.getHeader()->id=dns_random_uint16(); + pw.getHeader()->rd = true; + pw.getHeader()->qr = false; + pw.getHeader()->id = dns_random_uint16(); string qpacket((const char*)&packet[0], packet.size()); pw.startRecord(qname, QType::A, ttd); @@ -142,14 +143,13 @@ BOOST_AUTO_TEST_CASE(test_recPacketCacheSimplePost2038) { found = rpc.getResponsePacket(tag, qpacket, qname, QType::A, QClass::IN, future + 3601, &fpacket, &age, &qhash2); BOOST_CHECK_EQUAL(found, false); - packet.clear(); - qname+=DNSName("co.uk"); + qname += DNSName("co.uk"); DNSPacketWriter pw2(packet, qname, QType::A); - pw2.getHeader()->rd=true; - pw2.getHeader()->qr=false; - pw2.getHeader()->id=dns_random_uint16(); + pw2.getHeader()->rd = true; + pw2.getHeader()->qr = false; + pw2.getHeader()->id = dns_random_uint16(); qpacket.assign((const char*)&packet[0], packet.size()); found = rpc.getResponsePacket(tag, qpacket, future, &fpacket, &age, &qhash); @@ -161,7 +161,8 @@ BOOST_AUTO_TEST_CASE(test_recPacketCacheSimplePost2038) { BOOST_CHECK_EQUAL(rpc.size(), 0U); } -BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) { +BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) +{ /* Insert a response with tag1, the exact same query with a different tag should lead to a miss. Inserting a different response with the second tag should not override the first one, and we should get a hit for the @@ -169,23 +170,23 @@ BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) { */ RecursorPacketCache rpc; string fpacket; - const unsigned int tag1=0; - const unsigned int tag2=42; - uint32_t age=0; - uint32_t qhash=0; - uint32_t temphash=0; - uint32_t ttd=3600; + const unsigned int tag1 = 0; + const unsigned int tag2 = 42; + uint32_t age = 0; + uint32_t qhash = 0; + uint32_t temphash = 0; + uint32_t ttd = 3600; BOOST_CHECK_EQUAL(rpc.size(), 0U); - ::arg().set("rng")="auto"; - ::arg().set("entropy-source")="/dev/urandom"; + ::arg().set("rng") = "auto"; + ::arg().set("entropy-source") = "/dev/urandom"; DNSName qname("www.powerdns.com"); vector packet; DNSPacketWriter pw(packet, qname, QType::A); - pw.getHeader()->rd=true; - pw.getHeader()->qr=false; - pw.getHeader()->id=dns_random_uint16(); + pw.getHeader()->rd = true; + pw.getHeader()->qr = false; + pw.getHeader()->id = dns_random_uint16(); string qpacket(reinterpret_cast(&packet[0]), packet.size()); pw.startRecord(qname, QType::A, ttd); @@ -281,27 +282,28 @@ BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) { BOOST_CHECK_EQUAL(fpacket, r2packet); } -BOOST_AUTO_TEST_CASE(test_recPacketCache_TCP) { +BOOST_AUTO_TEST_CASE(test_recPacketCache_TCP) +{ /* Insert a response with UDP, the exact same query with a TCP flag should lead to a miss. */ RecursorPacketCache rpc; string fpacket; - uint32_t age=0; - uint32_t qhash=0; - uint32_t temphash=0; - uint32_t ttd=3600; + uint32_t age = 0; + uint32_t qhash = 0; + uint32_t temphash = 0; + uint32_t ttd = 3600; BOOST_CHECK_EQUAL(rpc.size(), 0U); - ::arg().set("rng")="auto"; - ::arg().set("entropy-source")="/dev/urandom"; + ::arg().set("rng") = "auto"; + ::arg().set("entropy-source") = "/dev/urandom"; DNSName qname("www.powerdns.com"); vector packet; DNSPacketWriter pw(packet, qname, QType::A); - pw.getHeader()->rd=true; - pw.getHeader()->qr=false; - pw.getHeader()->id=dns_random_uint16(); + pw.getHeader()->rd = true; + pw.getHeader()->qr = false; + pw.getHeader()->id = dns_random_uint16(); string qpacket(reinterpret_cast(&packet[0]), packet.size()); pw.startRecord(qname, QType::A, ttd); diff --git a/pdns/test-svc_records_cc.cc b/pdns/test-svc_records_cc.cc index a096cc68c1ab..98b41d25b7de 100644 --- a/pdns/test-svc_records_cc.cc +++ b/pdns/test-svc_records_cc.cc @@ -73,7 +73,6 @@ BOOST_AUTO_TEST_CASE(test_SvcParam_ctor_no_value) { BOOST_AUTO_TEST_CASE(test_SvcParam_ctor_value) { string val = "foobar"; auto base64val = Base64Encode(val); - SvcParam param; BOOST_CHECK_THROW(SvcParam(SvcParam::mandatory, val), std::invalid_argument); BOOST_CHECK_THROW(SvcParam(SvcParam::alpn, val), std::invalid_argument); @@ -82,6 +81,7 @@ BOOST_AUTO_TEST_CASE(test_SvcParam_ctor_value) { BOOST_CHECK_THROW(SvcParam(SvcParam::ipv4hint, val), std::invalid_argument); BOOST_CHECK_THROW(SvcParam(SvcParam::ipv6hint, val), std::invalid_argument); + SvcParam param(SvcParam::keyFromString("no-default-alpn")); BOOST_CHECK_NO_THROW(param = SvcParam(SvcParam::ech, base64val)); BOOST_CHECK_EQUAL(param.getECH(), base64val); BOOST_CHECK_THROW(param.getValue(), std::invalid_argument); @@ -118,7 +118,7 @@ BOOST_AUTO_TEST_CASE(test_SvcParam_ctor_set_string_value) { set mandatoryVal = {"alpn", "key666"}; set mandatoryExpected = {SvcParam::alpn, (SvcParam::SvcParamKey)666}; - SvcParam param; + SvcParam param(SvcParam::keyFromString("no-default-alpn")); BOOST_CHECK_NO_THROW(param = SvcParam(SvcParam::keyFromString("mandatory"), std::move(mandatoryVal))); auto retval = param.getMandatory(); @@ -141,7 +141,7 @@ BOOST_AUTO_TEST_CASE(test_SvcParam_ctor_vector_string_value) { BOOST_CHECK_THROW(SvcParam(SvcParam::ech, std::move(val)), std::invalid_argument); BOOST_CHECK_THROW(SvcParam(SvcParam::ipv6hint, std::move(val)), std::invalid_argument); - SvcParam param; + SvcParam param(SvcParam::keyFromString("no-default-alpn")); BOOST_CHECK_NO_THROW(param = SvcParam(SvcParam::keyFromString("alpn"), std::move(val))); auto alpns = param.getALPN(); @@ -176,7 +176,7 @@ BOOST_AUTO_TEST_CASE(test_SvcParam_ctor_set_comboaddress_value) { BOOST_CHECK_THROW(SvcParam(SvcParam::ipv6hint, std::move(mixedVal)), std::invalid_argument); BOOST_CHECK_THROW(SvcParam(SvcParam::ipv4hint, std::move(mixedVal)), std::invalid_argument); - SvcParam param; + SvcParam param(SvcParam::keyFromString("no-default-alpn")); BOOST_CHECK_NO_THROW(param = SvcParam(SvcParam::ipv4hint, std::move(v4Val))); auto retval = param.getIPHints(); @@ -208,7 +208,7 @@ BOOST_AUTO_TEST_CASE(test_SvcParam_ctor_uint16_value) { BOOST_CHECK_THROW(SvcParam(SvcParam::ech, port), std::invalid_argument); BOOST_CHECK_THROW(SvcParam(SvcParam::ipv6hint, port), std::invalid_argument); - SvcParam param; + SvcParam param(SvcParam::keyFromString("no-default-alpn")); BOOST_CHECK_NO_THROW(param = SvcParam(SvcParam::port, port)); BOOST_CHECK_EQUAL(param.getPort(), port); BOOST_CHECK_THROW(param.getMandatory(), std::invalid_argument); diff --git a/pdns/test-ueberbackend_cc.cc b/pdns/test-ueberbackend_cc.cc index fdfb1e6c5d2b..a5c143e22e10 100644 --- a/pdns/test-ueberbackend_cc.cc +++ b/pdns/test-ueberbackend_cc.cc @@ -137,8 +137,13 @@ class SimpleBackend : public DNSBackend findZone(qdomain, zoneId, d_records, d_currentZone); if (d_records) { - if (qdomain == DNSName("geo.powerdns.com.") && pkt_p != nullptr && pkt_p->getRealRemote() == Netmask("192.0.2.1")) { - d_currentScopeMask = 32; + if (qdomain == DNSName("geo.powerdns.com.") && pkt_p != nullptr) { + if (pkt_p->getRealRemote() == Netmask("192.0.2.1")) { + d_currentScopeMask = 32; + } + else if (pkt_p->getRealRemote() == Netmask("198.51.100.1")) { + d_currentScopeMask = 24; + } } auto& idx = d_records->get(); @@ -337,7 +342,7 @@ struct UeberBackendSetupArgFixture { ::arg().set("query-cache-ttl")="0"; ::arg().set("negquery-cache-ttl")="0"; ::arg().set("consistent-backends")="no"; - QC.cleanup(); + QC.purge(); ::arg().set("zone-cache-refresh-interval")="0"; g_zoneCache.clear(); BackendMakers().clear(); @@ -354,7 +359,8 @@ static void testWithoutThenWithAuthCache(std::function f /* disable the cache */ ::arg().set("query-cache-ttl")="0"; ::arg().set("negquery-cache-ttl")="0"; - QC.cleanup(); + QC.purge(); + QC.setMaxEntries(0); /* keep zone cache disabled */ ::arg().set("zone-cache-refresh-interval")="0"; g_zoneCache.clear(); @@ -367,7 +373,8 @@ static void testWithoutThenWithAuthCache(std::function f /* enable the cache */ ::arg().set("query-cache-ttl")="20"; ::arg().set("negquery-cache-ttl")="60"; - QC.cleanup(); + QC.purge(); + QC.setMaxEntries(100000); /* keep zone cache disabled */ ::arg().set("zone-cache-refresh-interval")="0"; g_zoneCache.clear(); @@ -391,7 +398,8 @@ static void testWithoutThenWithZoneCache(std::function f /* keep auth caches disabled */ ::arg().set("query-cache-ttl")="0"; ::arg().set("negquery-cache-ttl")="0"; - QC.cleanup(); + QC.purge(); + QC.setMaxEntries(0); UeberBackend ub; func(ub); @@ -404,7 +412,8 @@ static void testWithoutThenWithZoneCache(std::function f /* keep auth caches disabled */ ::arg().set("query-cache-ttl")="0"; ::arg().set("negquery-cache-ttl")="0"; - QC.cleanup(); + QC.purge(); + QC.setMaxEntries(0); UeberBackend ub; ub.updateZoneCache(); @@ -526,11 +535,11 @@ BOOST_AUTO_TEST_CASE(test_simple) { BOOST_REQUIRE_EQUAL(records.size(), 1U); checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 32, true); // and that we don't get the same result for a different client - remote = ComboAddress("192.0.2.2"); + remote = ComboAddress("198.51.100.1"); pkt.setRemote(&remote); records = getRecords(ub, DNSName("geo.powerdns.com."), QType::ANY, 1, &pkt); BOOST_REQUIRE_EQUAL(records.size(), 1U); - checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 0, true); + checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 24, true); } }; @@ -673,11 +682,11 @@ BOOST_AUTO_TEST_CASE(test_multi_backends_separate_zones) { BOOST_REQUIRE_EQUAL(records.size(), 1U); checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 32, true); // and that we don't get the same result for a different client - remote = ComboAddress("192.0.2.2"); + remote = ComboAddress("198.51.100.1"); pkt.setRemote(&remote); records = getRecords(ub, DNSName("geo.powerdns.com."), QType::ANY, 1, &pkt); BOOST_REQUIRE_EQUAL(records.size(), 1U); - checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 0, true); + checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 24, true); } }; @@ -799,11 +808,11 @@ BOOST_AUTO_TEST_CASE(test_multi_backends_overlay) { BOOST_REQUIRE_EQUAL(records.size(), 1U); checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 32, true); // and that we don't get the same result for a different client - remote = ComboAddress("192.0.2.2"); + remote = ComboAddress("198.51.100.1"); pkt.setRemote(&remote); records = getRecords(ub, DNSName("geo.powerdns.com."), QType::ANY, 1, &pkt); BOOST_REQUIRE_EQUAL(records.size(), 1U); - checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 0, true); + checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 24, true); } }; @@ -922,11 +931,11 @@ BOOST_AUTO_TEST_CASE(test_multi_backends_overlay_name) { BOOST_REQUIRE_EQUAL(records.size(), 1U); checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 32, true); // and that we don't get the same result for a different client - remote = ComboAddress("192.0.2.2"); + remote = ComboAddress("198.51.100.1"); pkt.setRemote(&remote); records = getRecords(ub, DNSName("geo.powerdns.com."), QType::ANY, 1, &pkt); BOOST_REQUIRE_EQUAL(records.size(), 1U); - checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 0, true); + checkRecordExists(records, DNSName("geo.powerdns.com."), QType::A, 1, 24, true); } }; @@ -1055,8 +1064,9 @@ BOOST_AUTO_TEST_CASE(test_multi_backends_best_soa) { BOOST_REQUIRE(ub.getAuth(DNSName("2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa."), QType::PTR, &sd)); BOOST_CHECK_EQUAL(sd.qname.toString(), "d.0.1.0.0.2.ip6.arpa."); BOOST_CHECK_EQUAL(sd.domain_id, 1); - // check that only one auth lookup occurred to this backend - BOOST_CHECK_EQUAL(sbba->d_authLookupCount, 1U); + + // check that at most one auth lookup occurred to this backend (O with caching enabled) + BOOST_CHECK_LE(sbba->d_authLookupCount, 1U); } }; diff --git a/pdns/test-zoneparser_tng_cc.cc b/pdns/test-zoneparser_tng_cc.cc index 665710944355..751645b22899 100644 --- a/pdns/test-zoneparser_tng_cc.cc +++ b/pdns/test-zoneparser_tng_cc.cc @@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(test_tng_record_types) { BOOST_CHECK_EQUAL(rr.qtype.toString(), type); if (rr.qtype == QType::SOA) continue; // FIXME400 remove trailing dots from data - if (*(rr.content.rbegin()) != '.' && *(data.rbegin()) == '.') + if (*(rr.content.rbegin()) != '.' && *(data.rbegin()) == '.') BOOST_CHECK_EQUAL(rr.content, std::string(data.begin(),data.end()-1)); else BOOST_CHECK_EQUAL(rr.content, data); @@ -116,6 +116,33 @@ BOOST_AUTO_TEST_CASE(test_tng_record_generate) { BOOST_CHECK_EQUAL(rr.qtype.toString(), "A"); BOOST_CHECK_EQUAL(rr.content, "1.2.3.4"); } + { + DNSResourceRecord rr; + BOOST_CHECK(!zp.get(rr)); + } + } + + { + /* GENERATE with a larger initial counter and a large stop */ + ZoneParserTNG zp(std::vector({"$GENERATE 4294967294-4294967295/2 $ 86400 IN A 1.2.3.4"}), DNSName("unit2.test")); + + const vector expected = { + "4294967294.unit2.test.", + }; + + for (auto const & exp : expected) { + DNSResourceRecord rr; + zp.get(rr); + BOOST_CHECK_EQUAL(rr.qname.toString(), exp); + BOOST_CHECK_EQUAL(rr.ttl, 86400U); + BOOST_CHECK_EQUAL(rr.qclass, 1U); + BOOST_CHECK_EQUAL(rr.qtype.toString(), "A"); + BOOST_CHECK_EQUAL(rr.content, "1.2.3.4"); + } + { + DNSResourceRecord rr; + BOOST_CHECK(!zp.get(rr)); + } } { @@ -139,6 +166,40 @@ BOOST_AUTO_TEST_CASE(test_tng_record_generate) { BOOST_CHECK_THROW(zp.get(rr), std::exception); } + { + /* test invalid generate parameters: negative counter */ + ZoneParserTNG zp(std::vector({"$GENERATE -1-4/1 $.${1,2,o}.${3,4,d}.${5,6,X}.${7,8,x} 86400 IN A 1.2.3.4"}), DNSName("test")); + DNSResourceRecord rr; + BOOST_CHECK_THROW(zp.get(rr), std::exception); + } + { + /* test invalid generate parameters: counter out of bounds */ + ZoneParserTNG zp(std::vector({"$GENERATE 4294967296-4/1 $.${1,2,o}.${3,4,d}.${5,6,X}.${7,8,x} 86400 IN A 1.2.3.4"}), DNSName("test")); + DNSResourceRecord rr; + BOOST_CHECK_THROW(zp.get(rr), std::exception); + } + + { + /* test invalid generate parameters: negative stop */ + ZoneParserTNG zp(std::vector({"$GENERATE 0--4/1 $.${1,2,o}.${3,4,d}.${5,6,X}.${7,8,x} 86400 IN A 1.2.3.4"}), DNSName("test")); + DNSResourceRecord rr; + BOOST_CHECK_THROW(zp.get(rr), std::exception); + } + + { + /* test invalid generate parameters: stop out of bounds */ + ZoneParserTNG zp(std::vector({"$GENERATE 0-4294967296/1 $.${1,2,o}.${3,4,d}.${5,6,X}.${7,8,x} 86400 IN A 1.2.3.4"}), DNSName("test")); + DNSResourceRecord rr; + BOOST_CHECK_THROW(zp.get(rr), std::exception); + } + + { + /* test invalid generate parameters: negative step */ + ZoneParserTNG zp(std::vector({"$GENERATE 0-4/-1 $.${1,2,o}.${3,4,d}.${5,6,X}.${7,8,x} 86400 IN A 1.2.3.4"}), DNSName("test")); + DNSResourceRecord rr; + BOOST_CHECK_THROW(zp.get(rr), std::exception); + } + { /* test invalid generate parameters: no offset */ ZoneParserTNG zp(std::vector({"$GENERATE 0-4/1 $.${}.${3,4,d}.${5,6,X}.${7,8,x} 86400 IN A 1.2.3.4"}), DNSName("test")); diff --git a/pdns/threadname.cc b/pdns/threadname.cc index 249248a62387..4318117d0a2d 100644 --- a/pdns/threadname.cc +++ b/pdns/threadname.cc @@ -39,7 +39,7 @@ #include "threadname.hh" -void setThreadName(const std::string& threadName) { +static int trySetThreadName(const std::string& threadName) { int retval = 0; #ifdef HAVE_PTHREAD_SETNAME_NP_2 @@ -58,6 +58,16 @@ void setThreadName(const std::string& threadName) { retval = pthread_setname_np(pthread_self(), threadName.c_str(), nullptr); #endif + return retval; +} + +void setThreadName(const std::string& threadName) { + int retval = trySetThreadName(threadName); + if (retval == ERANGE) { + const std::string shortThreadName(threadName.substr(0, 15)); + retval = trySetThreadName(shortThreadName); + } + if (retval != 0) { #ifdef DNSDIST warnlog("Could not set thread name %s for thread: %s", threadName, strerror(retval)); diff --git a/pdns/toysdig.cc b/pdns/toysdig.cc index b2e93c30a675..80a63291c28c 100644 --- a/pdns/toysdig.cc +++ b/pdns/toysdig.cc @@ -56,7 +56,7 @@ class TCPResolver : public boost::noncopyable throw PDNSException("EOF on TCP read"); len=ntohs(len); - std::unique_ptr creply(new char[len]); + auto creply = std::make_unique(len); int n=0; int numread; while(nd_signer<<"/"<d_tag<d_signer, keys); cerr<<"! state = "<UeberBackend::instances; -std::mutex UeberBackend::instances_lock; +LockGuarded> UeberBackend::d_instances; // initially we are blocked bool UeberBackend::d_go=false; @@ -285,9 +284,9 @@ void UeberBackend::updateZoneCache() { for (vector::iterator i = backends.begin(); i != backends.end(); ++i ) { vector zones; - (*i)->getAllDomains(&zones, true); + (*i)->getAllDomains(&zones, false, true); for(auto& di: zones) { - zone_indices.push_back({std::move(di.zone), (int)di.id}); // this cast should not be necessary + zone_indices.emplace_back(std::move(di.zone), (int)di.id); // this cast should not be necessary } } g_zoneCache.replace(zone_indices); @@ -346,26 +345,38 @@ bool UeberBackend::getAuth(const DNSName &target, const QType& qtype, SOAData* s bool found = false; int cstat; DNSName shorter(target); - vector > bestmatch (backends.size(), make_pair(target.wirelength()+1, SOAData())); + vector > bestmatch (backends.size(), pair(target.wirelength()+1, SOAData())); do { int zoneId{-1}; if(cachedOk && g_zoneCache.isEnabled()) { if (g_zoneCache.getEntry(shorter, zoneId)) { // Zone exists in zone cache, directly look up SOA. - // XXX: this code path and the cache lookup below should be merged; but that needs the code path below to also use ANY. - // Or it should just also use lookup(). DNSZoneRecord zr; lookup(QType(QType::SOA), shorter, zoneId, nullptr); if (!get(zr)) { - // zone has somehow vanished DLOG(g_log << Logger::Info << "Backend returned no SOA for zone '" << shorter.toLogString() << "', which it reported as existing " << endl); continue; } if (zr.dr.d_name != shorter) { throw PDNSException("getAuth() returned an SOA for the wrong zone. Zone '"+zr.dr.d_name.toLogString()+"' is not equal to looked up zone '"+shorter.toLogString()+"'"); } + // fill sd sd->qname = zr.dr.d_name; - fillSOAData(zr, *sd); + try { + fillSOAData(zr, *sd); + } + catch (...) { + g_log << Logger::Warning << "Backend returned a broken SOA for zone '" << shorter.toLogString() << "'" << endl; + while (get(zr)) + ; + continue; + } + if (backends.size() == 1) { + sd->db = *backends.begin(); + } + else { + sd->db = nullptr; + } // leave database handle in a consistent state while (get(zr)) ; @@ -520,8 +531,7 @@ bool UeberBackend::superMasterBackend(const string &ip, const DNSName &domain, c UeberBackend::UeberBackend(const string &pname) { { - std::lock_guard l(instances_lock); - instances.push_back(this); // report to the static list of ourself + d_instances.lock()->push_back(this); // report to the static list of ourself } d_negcached=false; @@ -542,9 +552,9 @@ static void del(DNSBackend* d) void UeberBackend::cleanup() { { - std::lock_guard l(instances_lock); - remove(instances.begin(),instances.end(),this); - instances.resize(instances.size()-1); + auto instances = d_instances.lock(); + remove(instances->begin(), instances->end(), this); + instances->resize(instances->size()-1); } for_each(backends.begin(),backends.end(),del); @@ -667,10 +677,11 @@ void UeberBackend::lookup(const QType &qtype,const DNSName &qname, int zoneId, D d_handle.parent=this; } -void UeberBackend::getAllDomains(vector *domains, bool include_disabled) { +void UeberBackend::getAllDomains(vector* domains, bool getSerial, bool include_disabled) +{ for (auto & backend : backends) { - backend->getAllDomains(domains, include_disabled); + backend->getAllDomains(domains, getSerial, include_disabled); } } diff --git a/pdns/ueberbackend.hh b/pdns/ueberbackend.hh index b3b688aac68a..53e1e8de63d3 100644 --- a/pdns/ueberbackend.hh +++ b/pdns/ueberbackend.hh @@ -31,6 +31,7 @@ #include "dnspacket.hh" #include "dnsbackend.hh" +#include "lock.hh" #include "namespaces.hh" /** This is a very magic backend that allows us to load modules dynamically, @@ -53,8 +54,7 @@ public: /** Tracks all created UeberBackend instances for us. We use this vector to notify existing threads of new modules */ - static vectorinstances; - static std::mutex instances_lock; + static LockGuarded> d_instances; static bool loadmodule(const string &name); static bool loadModules(const vector& modules, const string& path); @@ -101,7 +101,7 @@ public: /** Load SOA info from backends, ignoring the cache.*/ bool getSOAUncached(const DNSName &domain, SOAData &sd); bool get(DNSZoneRecord &r); - void getAllDomains(vector *domains, bool include_disabled=false); + void getAllDomains(vector* domains, bool getSerial, bool include_disabled); void getUnfreshSlaveInfos(vector* domains); void getUpdatedMasters(vector* domains); diff --git a/pdns/validate-recursor.cc b/pdns/validate-recursor.cc index c430d3d19e4b..4c2bb1086a4c 100644 --- a/pdns/validate-recursor.cc +++ b/pdns/validate-recursor.cc @@ -10,14 +10,16 @@ DNSSECMode g_dnssecmode{DNSSECMode::ProcessNoValidate}; bool g_dnssecLogBogus; -bool checkDNSSECDisabled() { +bool checkDNSSECDisabled() +{ return warnIfDNSSECDisabled(""); } -bool warnIfDNSSECDisabled(const string& msg) { - if(g_dnssecmode == DNSSECMode::Off) { +bool warnIfDNSSECDisabled(const string& msg) +{ + if (g_dnssecmode == DNSSECMode::Off) { if (!msg.empty()) - g_log< &dsAnchors) { - map newDSAnchors; +bool updateTrustAnchorsFromFile(const std::string& fname, map& dsAnchors) +{ + map newDSAnchors; try { auto zp = ZoneParserTNG(fname); zp.disableGenerate(); DNSResourceRecord rr; DNSRecord dr; - while(zp.get(rr)) { + while (zp.get(rr)) { dr = DNSRecord(rr); if (rr.qtype == QType::DS) { auto dsr = getRR(dr); @@ -55,21 +58,21 @@ bool updateTrustAnchorsFromFile(const std::string &fname, map if (rr.qtype == QType::DNSKEY) { auto dnskeyr = getRR(dr); if (dnskeyr == nullptr) { - throw PDNSException("Unable to parse DNSKEY record '" + rr.qname.toString() + " " + rr.getZoneRepresentation() +"'"); + throw PDNSException("Unable to parse DNSKEY record '" + rr.qname.toString() + " " + rr.getZoneRepresentation() + "'"); } auto dsr = makeDSFromDNSKey(rr.qname, *dnskeyr, DNSSECKeeper::DIGEST_SHA256); newDSAnchors[rr.qname].insert(dsr); } } if (dsAnchors == newDSAnchors) { - g_log< servfail */ -enum class DNSSECMode { Off, Process, ProcessNoValidate, ValidateForLog, ValidateAll }; +enum class DNSSECMode +{ + Off, + Process, + ProcessNoValidate, + ValidateForLog, + ValidateAll +}; extern DNSSECMode g_dnssecmode; extern bool g_dnssecLogBogus; @@ -39,4 +46,4 @@ bool checkDNSSECDisabled(); bool warnIfDNSSECDisabled(const string& msg); vState increaseDNSSECStateCounter(const vState& state); vState increaseXDNSSECStateCounter(const vState& state); -bool updateTrustAnchorsFromFile(const std::string &fname, map &dsAnchors); +bool updateTrustAnchorsFromFile(const std::string& fname, map& dsAnchors); diff --git a/pdns/validate.cc b/pdns/validate.cc index eda299fbd6a0..1613abbec66a 100644 --- a/pdns/validate.cc +++ b/pdns/validate.cc @@ -512,6 +512,16 @@ dState getDenial(const cspmap_t &validrrsets, const DNSName& qname, const uint16 continue; } + /* The NSEC is either a delegation one, from the parent zone, and + * must have the NS bit set but not the SOA one, or a regular NSEC + * either at apex (signer == owner) or with the SOA or NS bits clear. + */ + const bool notApex = signer.countLabels() < owner.countLabels(); + if (notApex && nsec->isSet(QType::NS) && nsec->isSet(QType::SOA)) { + LOG("However, that NSEC is not at the apex and has both the NS and the SOA bits set!"<isSet(QType::NS)) { - LOG("However, no NS record exists at this level!"<isSet(QType::NS)) { + LOG("However, no NS record exists at this level!"<isSet(QType::NS) && nsec3->isSet(QType::SOA)) { + LOG("However, that NSEC3 is not at the apex and has both the NS and the SOA bits set!"<isSet(QType::NS)) { - LOG("However, no NS record exists at this level!"<isSet(QType::NS)) { + LOG("However, no NS record exists at this level!"<& sig return sig->d_siginception - g_signatureInceptionSkew <= now; } -static bool checkSignatureWithKey(time_t now, const shared_ptr sig, const shared_ptr key, const std::string& msg) +static bool checkSignatureWithKey(time_t now, const shared_ptr sig, const shared_ptr key, const std::string& msg, vState& ede) { bool result = false; try { @@ -921,13 +957,18 @@ static bool checkSignatureWithKey(time_t now, const shared_ptrd_algorithm, key->d_key); result = dke->verify(msg, sig->d_signature); LOG("signature by key with tag "<d_tag<<" and algorithm "<d_algorithm)<<" was " << (result ? "" : "NOT ")<<"valid"<d_siginception - g_signatureInceptionSkew > now) ? "not yet valid" : "expired")<<" (inception: "<d_siginception<<", inception skew: "<d_sigexpire<<", now: "<d_siginception - g_signatureInceptionSkew) > now) ? vState::BogusSignatureNotYetValid : vState::BogusSignatureExpired; + LOG("Signature is "<<(ede == vState::BogusSignatureNotYetValid ? "not yet valid" : "expired")<<" (inception: "<d_siginception<<", inception skew: "<d_sigexpire<<", now: "<second).find("basic ") == 0) { string cookie = header->second.substr(6); @@ -66,22 +66,32 @@ bool HttpRequest::compareAuthorization(const string &expected_password) vector cparts; stringtok(cparts, plain, ":"); - // this gets rid of terminating zeros - auth_ok = (cparts.size()==2 && (0==strcmp(cparts[1].c_str(), expected_password.c_str()))); + auth_ok = (cparts.size() == 2 && credentials.matches(cparts[1].c_str())); } return auth_ok; } -bool HttpRequest::compareHeader(const string &header_name, const string &expected_value) +bool HttpRequest::compareHeader(const string &header_name, const string &expected_value) const { - YaHTTP::strstr_map_t::iterator header = headers.find(header_name); - if (header == headers.end()) + auto header = headers.find(header_name); + if (header == headers.end()) { return false; + } // this gets rid of terminating zeros return (0==strcmp(header->second.c_str(), expected_value.c_str())); } +bool HttpRequest::compareHeader(const string &header_name, const CredentialsHolder& credentials) const +{ + auto header = headers.find(header_name); + if (header == headers.end()) { + return false; + } + + return credentials.matches(header->second); +} + void HttpResponse::setPlainBody(const string& document) { this->headers["Content-Type"] = "text/plain; charset=utf-8"; @@ -153,16 +163,16 @@ void WebServer::apiWrapper(const WebServer::HandlerFunction& handler, HttpReques resp->headers["access-control-allow-origin"] = "*"; - if (d_apikey.empty()) { + if (!d_apikey) { g_log<logprefix<<"HTTP API Request \"" << req->url.path << "\": Authentication failed, API Key missing in config" << endl; throw HttpUnauthorizedException("X-API-Key"); } - bool auth_ok = req->compareHeader("x-api-key", d_apikey) || req->getvars["api-key"] == d_apikey; + bool auth_ok = req->compareHeader("x-api-key", *d_apikey) || d_apikey->matches(req->getvars["api-key"]); if (!auth_ok && allowPassword) { - if (!d_webserverPassword.empty()) { - auth_ok = req->compareAuthorization(d_webserverPassword); + if (d_webserverPassword) { + auth_ok = req->compareAuthorization(*d_webserverPassword); } else { auth_ok = true; } @@ -205,8 +215,8 @@ void WebServer::registerApiHandler(const string& url, const HandlerFunction& han } void WebServer::webWrapper(const WebServer::HandlerFunction& handler, HttpRequest* req, HttpResponse* resp) { - if (!d_webserverPassword.empty()) { - bool auth_ok = req->compareAuthorization(d_webserverPassword); + if (d_webserverPassword) { + bool auth_ok = req->compareAuthorization(*d_webserverPassword); if (!auth_ok) { g_log<logprefix<<"HTTP Request \"" << req->url.path << "\": Web Authentication failed" << endl; throw HttpUnauthorizedException("Basic"); diff --git a/pdns/webserver.hh b/pdns/webserver.hh index 858b56e9eb77..d8707e09149a 100644 --- a/pdns/webserver.hh +++ b/pdns/webserver.hh @@ -25,7 +25,10 @@ #include #include #include + #include "json11.hpp" + +#include "credentials.hh" #include "namespaces.hh" #include "sstuff.hh" @@ -42,8 +45,9 @@ public: json11::Json json(); // checks password _only_. - bool compareAuthorization(const string &expected_password); - bool compareHeader(const string &header_name, const string &expected_value); + bool compareAuthorization(const CredentialsHolder& expectedCredentials) const; + bool compareHeader(const string &header_name, const CredentialsHolder& expectedCredentials) const; + bool compareHeader(const string &header_name, const string &expected_value) const; }; class HttpResponse: public YaHTTP::Response { @@ -159,12 +163,22 @@ public: WebServer(string listenaddress, int port); virtual ~WebServer() { }; - void setApiKey(const string &apikey) { - d_apikey = apikey; + void setApiKey(const string &apikey, bool hashPlaintext) { + if (!apikey.empty()) { + d_apikey = make_unique(std::string(apikey), hashPlaintext); + } + else { + d_apikey.reset(); + } } - void setPassword(const string &password) { - d_webserverPassword = password; + void setPassword(const string &password, bool hashPlaintext) { + if (!password.empty()) { + d_webserverPassword = make_unique(std::string(password), hashPlaintext); + } + else { + d_webserverPassword.reset(); + } } void setMaxBodySize(ssize_t s) { // in megabytes @@ -227,15 +241,15 @@ protected: return std::make_shared(d_listenaddress, d_port); } + void apiWrapper(const WebServer::HandlerFunction& handler, HttpRequest* req, HttpResponse* resp, bool allowPassword); + void webWrapper(const WebServer::HandlerFunction& handler, HttpRequest* req, HttpResponse* resp); + string d_listenaddress; int d_port; - string d_password; std::shared_ptr d_server; - std::string d_apikey; - void apiWrapper(const WebServer::HandlerFunction& handler, HttpRequest* req, HttpResponse* resp, bool allowPassword); - std::string d_webserverPassword; - void webWrapper(const WebServer::HandlerFunction& handler, HttpRequest* req, HttpResponse* resp); + std::unique_ptr d_apikey{nullptr}; + std::unique_ptr d_webserverPassword{nullptr}; ssize_t d_maxbodysize; // in bytes diff --git a/pdns/ws-api.cc b/pdns/ws-api.cc index cea1abd78e40..470035386e9e 100644 --- a/pdns/ws-api.cc +++ b/pdns/ws-api.cc @@ -117,6 +117,20 @@ void apiDiscovery(HttpRequest* req, HttpResponse* resp) { resp->setJsonBody(doc); } +void apiDiscoveryV1(HttpRequest* req, HttpResponse* resp) { + if(req->method != "GET") + throw HttpMethodNotAllowedException(); + + Json version1 = Json::object { + { "server_url", "/api/v1/servers{/server}" }, + { "api_features", Json::array {} } + }; + Json doc = Json::array { version1 }; + + resp->setJsonBody(doc); + +} + void apiServer(HttpRequest* req, HttpResponse* resp) { if(req->method != "GET") throw HttpMethodNotAllowedException(); diff --git a/pdns/ws-api.hh b/pdns/ws-api.hh index d242110afda8..461411b1ace2 100644 --- a/pdns/ws-api.hh +++ b/pdns/ws-api.hh @@ -24,6 +24,7 @@ #include "webserver.hh" void apiDiscovery(HttpRequest* req, HttpResponse* resp); +void apiDiscoveryV1(HttpRequest* req, HttpResponse* resp); void apiServer(HttpRequest* req, HttpResponse* resp); void apiServerDetail(HttpRequest* req, HttpResponse* resp); void apiServerConfig(HttpRequest* req, HttpResponse* resp); diff --git a/pdns/ws-auth.cc b/pdns/ws-auth.cc index b1a39a86ded6..6d691264d29b 100644 --- a/pdns/ws-auth.cc +++ b/pdns/ws-auth.cc @@ -58,6 +58,10 @@ static void patchZone(UeberBackend& B, HttpRequest* req, HttpResponse* resp); static const std::set onlyOneEntryTypes = { QType::CNAME, QType::DNAME, QType::SOA }; // QTypes that MUST NOT be used with any other QType on the same name. static const std::set exclusiveEntryTypes = { QType::CNAME }; +// QTypes that MUST be at apex. +static const std::set atApexTypes = {QType::SOA, QType::DNSKEY}; +// QTypes that are NOT allowed at apex. +static const std::set nonApexTypes = {QType::DS}; AuthWebServer::AuthWebServer() : d_start(time(nullptr)), @@ -65,10 +69,10 @@ AuthWebServer::AuthWebServer() : d_min5(0), d_min1(0) { - if(arg().mustDo("webserver") || arg().mustDo("api")) { - d_ws = unique_ptr(new WebServer(arg()["webserver-address"], arg().asNum("webserver-port"))); - d_ws->setApiKey(arg()["api-key"]); - d_ws->setPassword(arg()["webserver-password"]); + if (arg().mustDo("webserver") || arg().mustDo("api")) { + d_ws = std::make_unique(arg()["webserver-address"], arg().asNum("webserver-port")); + d_ws->setApiKey(arg()["api-key"], arg().mustDo("webserver-hash-plaintext-credentials")); + d_ws->setPassword(arg()["webserver-password"], arg().mustDo("webserver-hash-plaintext-credentials")); d_ws->setLogLevel(arg()["webserver-loglevel"]); NetmaskGroup acl; @@ -1385,6 +1389,7 @@ static void gatherRecordsFromZone(const std::string& zonestring, vector& records) { +static void checkNewRecords(vector& records, const DNSName& zone) +{ sort(records.begin(), records.end(), [](const DNSResourceRecord& rec_a, const DNSResourceRecord& rec_b) -> bool { /* we need _strict_ weak ordering */ @@ -1437,6 +1443,15 @@ static void checkNewRecords(vector& records) { } } + if (rec.qname == zone) { + if (nonApexTypes.count(rec.qtype.getCode()) != 0) { + throw ApiException("Record " + rec.qname.toString() + " IN " + rec.qtype.toString() + " is not allowed at apex"); + } + } + else if (atApexTypes.count(rec.qtype.getCode()) != 0) { + throw ApiException("Record " + rec.qname.toString() + " IN " + rec.qtype.toString() + " is only allowed at apex"); + } + // Check if the DNSNames that should be hostnames, are hostnames try { checkHostnameCorrectness(rec); @@ -1704,7 +1719,7 @@ static void apiServerZones(HttpRequest* req, HttpResponse* resp) { } } - checkNewRecords(new_records); + checkNewRecords(new_records, zonename); if (boolFromJson(document, "dnssec", false)) { checkDefaultDNSSECAlgos(); @@ -1773,7 +1788,7 @@ static void apiServerZones(HttpRequest* req, HttpResponse* resp) { } } else { try { - B.getAllDomains(&domains, true); // incl. disabled + B.getAllDomains(&domains, true, true); // incl. serial and disabled } catch(const PDNSException &e) { throw HttpInternalServerErrorException("Could not retrieve all domain information: " + e.reason); } @@ -2032,7 +2047,7 @@ static void patchZone(UeberBackend& B, HttpRequest* req, HttpResponse* resp) { soa_edit_done = increaseSOARecord(rr, soa_edit_api_kind, soa_edit_kind); } } - checkNewRecords(new_records); + checkNewRecords(new_records, zonename); } if (replace_comments) { @@ -2190,7 +2205,7 @@ static void apiServerSearchData(HttpRequest* req, HttpResponse* resp) { map::iterator val; Json::array doc; - B.getAllDomains(&domains, true); + B.getAllDomains(&domains, false, true); for(const DomainInfo& di: domains) { @@ -2336,26 +2351,27 @@ void AuthWebServer::webThread() try { setThreadName("pdns/webserver"); if(::arg().mustDo("api")) { - d_ws->registerApiHandler("/api/v1/servers/localhost/cache/flush", &apiServerCacheFlush); - d_ws->registerApiHandler("/api/v1/servers/localhost/config", &apiServerConfig); - d_ws->registerApiHandler("/api/v1/servers/localhost/search-data", &apiServerSearchData); - d_ws->registerApiHandler("/api/v1/servers/localhost/statistics", &apiServerStatistics); - d_ws->registerApiHandler("/api/v1/servers/localhost/tsigkeys/", &apiServerTSIGKeyDetail); - d_ws->registerApiHandler("/api/v1/servers/localhost/tsigkeys", &apiServerTSIGKeys); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//axfr-retrieve", &apiServerZoneAxfrRetrieve); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//cryptokeys/", &apiZoneCryptokeys); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//cryptokeys", &apiZoneCryptokeys); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//export", &apiServerZoneExport); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//metadata/", &apiZoneMetadataKind); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//metadata", &apiZoneMetadata); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//notify", &apiServerZoneNotify); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones//rectify", &apiServerZoneRectify); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones/", &apiServerZoneDetail); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones", &apiServerZones); - d_ws->registerApiHandler("/api/v1/servers/localhost", &apiServerDetail); - d_ws->registerApiHandler("/api/v1/servers", &apiServer); - d_ws->registerApiHandler("/api/docs", &apiDocs); - d_ws->registerApiHandler("/api", &apiDiscovery); + d_ws->registerApiHandler("/api/v1/servers/localhost/cache/flush", apiServerCacheFlush); + d_ws->registerApiHandler("/api/v1/servers/localhost/config", apiServerConfig); + d_ws->registerApiHandler("/api/v1/servers/localhost/search-data", apiServerSearchData); + d_ws->registerApiHandler("/api/v1/servers/localhost/statistics", apiServerStatistics); + d_ws->registerApiHandler("/api/v1/servers/localhost/tsigkeys/", apiServerTSIGKeyDetail); + d_ws->registerApiHandler("/api/v1/servers/localhost/tsigkeys", apiServerTSIGKeys); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//axfr-retrieve", apiServerZoneAxfrRetrieve); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//cryptokeys/", apiZoneCryptokeys); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//cryptokeys", apiZoneCryptokeys); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//export", apiServerZoneExport); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//metadata/", apiZoneMetadataKind); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//metadata", apiZoneMetadata); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//notify", apiServerZoneNotify); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones//rectify", apiServerZoneRectify); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones/", apiServerZoneDetail); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones", apiServerZones); + d_ws->registerApiHandler("/api/v1/servers/localhost", apiServerDetail); + d_ws->registerApiHandler("/api/v1/servers", apiServer); + d_ws->registerApiHandler("/api/v1", apiDiscoveryV1); + d_ws->registerApiHandler("/api/docs", apiDocs); + d_ws->registerApiHandler("/api", apiDiscovery); } if (::arg().mustDo("webserver")) { d_ws->registerWebHandler("/style.css", [this](HttpRequest *req, HttpResponse *resp){cssfunction(req, resp);}); diff --git a/pdns/ws-recursor.cc b/pdns/ws-recursor.cc index 9df78ec35813..84feae928640 100644 --- a/pdns/ws-recursor.cc +++ b/pdns/ws-recursor.cc @@ -48,12 +48,12 @@ extern thread_local FDMultiplexer* t_fdm; using json11::Json; -void productServerStatisticsFetch(map& out) +void productServerStatisticsFetch(map& out) { auto stats = getAllStatsMap(StatComponent::API); - map ret; - for (const auto& entry: stats) { - ret.insert(make_pair(entry.first, entry.second.d_value)); + map ret; + for (const auto& entry : stats) { + ret.emplace(entry.first, entry.second.d_value); } out.swap(ret); } @@ -72,14 +72,14 @@ static void apiWriteConfigFile(const string& filebasename, const string& content string filename = ::arg()["api-config-dir"] + "/" + filebasename + ".conf"; ofstream ofconf(filename.c_str()); if (!ofconf) { - throw ApiException("Could not open config fragment file '"+filename+"' for writing: "+stringerror()); + throw ApiException("Could not open config fragment file '" + filename + "' for writing: " + stringerror()); } ofconf << "# Generated by pdns-recursor REST API, DO NOT EDIT" << endl; ofconf << content << endl; ofconf.close(); } -static void apiServerConfigAllowFrom(HttpRequest* req, HttpResponse* resp) +static void apiServerConfigACL(const std::string& aclType, HttpRequest* req, HttpResponse* resp) { if (req->method == "PUT") { Json document = req->json(); @@ -93,73 +93,88 @@ static void apiServerConfigAllowFrom(HttpRequest* req, HttpResponse* resp) for (auto value : jlist.array_items()) { try { nmg.addMask(value.string_value()); - } catch (const NetmaskException &e) { + } + catch (const NetmaskException& e) { throw ApiException(e.reason); } } ostringstream ss; - // Clear allow-from-file if set, so our changes take effect - ss << "allow-from-file=" << endl; + // Clear -from-file if set, so our changes take effect + ss << aclType << "-file=" << endl; - // Clear allow-from, and provide a "parent" value - ss << "allow-from=" << endl; - ss << "allow-from+=" << nmg.toString() << endl; + // Clear ACL setting, and provide a "parent" value + ss << aclType << "=" << endl; + ss << aclType << "+=" << nmg.toString() << endl; - apiWriteConfigFile("allow-from", ss.str()); + apiWriteConfigFile(aclType, ss.str()); parseACLs(); // fall through to GET - } else if (req->method != "GET") { + } + else if (req->method != "GET") { throw HttpMethodNotAllowedException(); } // Return currently configured ACLs vector entries; - t_allowFrom->toStringVector(&entries); + if (aclType == "allow-from") { + t_allowFrom->toStringVector(&entries); + } + else if (aclType == "allow-notify-from") { + t_allowNotifyFrom->toStringVector(&entries); + } - resp->setJsonBody(Json::object { - { "name", "allow-from" }, - { "value", entries }, + resp->setJsonBody(Json::object{ + {"name", aclType}, + {"value", entries}, }); } +static void apiServerConfigAllowFrom(HttpRequest* req, HttpResponse* resp) +{ + apiServerConfigACL("allow-from", req, resp); +} + +static void apiServerConfigAllowNotifyFrom(HttpRequest* req, HttpResponse* resp) +{ + apiServerConfigACL("allow-notify-from", req, resp); +} + static void fillZone(const DNSName& zonename, HttpResponse* resp) { auto iter = SyncRes::t_sstorage.domainmap->find(zonename); if (iter == SyncRes::t_sstorage.domainmap->end()) - throw ApiException("Could not find domain '"+zonename.toLogString()+"'"); + throw ApiException("Could not find domain '" + zonename.toLogString() + "'"); const SyncRes::AuthDomain& zone = iter->second; Json::array servers; - for(const ComboAddress& server : zone.d_servers) { + for (const ComboAddress& server : zone.d_servers) { servers.push_back(server.toStringWithPort()); } Json::array records; - for(const SyncRes::AuthDomain::records_t::value_type& dr : zone.d_records) { - records.push_back(Json::object { - { "name", dr.d_name.toString() }, - { "type", DNSRecordContent::NumberToType(dr.d_type) }, - { "ttl", (double)dr.d_ttl }, - { "content", dr.d_content->getZoneRepresentation() } - }); + for (const SyncRes::AuthDomain::records_t::value_type& dr : zone.d_records) { + records.push_back(Json::object{ + {"name", dr.d_name.toString()}, + {"type", DNSRecordContent::NumberToType(dr.d_type)}, + {"ttl", (double)dr.d_ttl}, + {"content", dr.d_content->getZoneRepresentation()}}); } // id is the canonical lookup key, which doesn't actually match the name (in some cases) string zoneId = apiZoneNameToId(iter->first); Json::object doc = { - { "id", zoneId }, - { "url", "/api/v1/servers/localhost/zones/" + zoneId }, - { "name", iter->first.toString() }, - { "kind", zone.d_servers.empty() ? "Native" : "Forwarded" }, - { "servers", servers }, - { "recursion_desired", zone.d_servers.empty() ? false : zone.d_rdForward }, - { "records", records } - }; + {"id", zoneId}, + {"url", "/api/v1/servers/localhost/zones/" + zoneId}, + {"name", iter->first.toString()}, + {"kind", zone.d_servers.empty() ? "Native" : "Forwarded"}, + {"servers", servers}, + {"recursion_desired", zone.d_servers.empty() ? false : zone.d_rdForward}, + {"records", records}}; resp->setJsonBody(doc); } @@ -181,32 +196,33 @@ static void doCreateZone(const Json document) if (kind == "NATIVE") { if (rd) throw ApiException("kind=Native and recursion_desired are mutually exclusive"); - if(!singleIPTarget.empty()) { + if (!singleIPTarget.empty()) { try { - ComboAddress rem(singleIPTarget); - if(rem.sin4.sin_family != AF_INET) - throw ApiException(""); - singleIPTarget = rem.toString(); + ComboAddress rem(singleIPTarget); + if (rem.sin4.sin_family != AF_INET) + throw ApiException(""); + singleIPTarget = rem.toString(); } - catch(...) { - throw ApiException("Single IP target '"+singleIPTarget+"' is invalid"); + catch (...) { + throw ApiException("Single IP target '" + singleIPTarget + "' is invalid"); } } string zonefilename = ::arg()["api-config-dir"] + "/" + confbasename + ".zone"; ofstream ofzone(zonefilename.c_str()); if (!ofzone) { - throw ApiException("Could not open '"+zonefilename+"' for writing: "+stringerror()); + throw ApiException("Could not open '" + zonefilename + "' for writing: " + stringerror()); } ofzone << "; Generated by pdns-recursor REST API, DO NOT EDIT" << endl; - ofzone << zonename << "\tIN\tSOA\tlocal.zone.\thostmaster."<status = 201; return; } - if(req->method != "GET") + if (req->method != "GET") throw HttpMethodNotAllowedException(); Json::array doc; - for(const SyncRes::domainmap_t::value_type& val : *SyncRes::t_sstorage.domainmap) { + for (const SyncRes::domainmap_t::value_type& val : *SyncRes::t_sstorage.domainmap) { const SyncRes::AuthDomain& zone = val.second; Json::array servers; - for(const ComboAddress& server : zone.d_servers) { + for (const ComboAddress& server : zone.d_servers) { servers.push_back(server.toStringWithPort()); } // id is the canonical lookup key, which doesn't actually match the name (in some cases) string zoneId = apiZoneNameToId(val.first); - doc.push_back(Json::object { - { "id", zoneId }, - { "url", "/api/v1/servers/localhost/zones/" + zoneId }, - { "name", val.first.toString() }, - { "kind", zone.d_servers.empty() ? "Native" : "Forwarded" }, - { "servers", servers }, - { "recursion_desired", zone.d_servers.empty() ? false : zone.d_rdForward } - }); + doc.push_back(Json::object{ + {"id", zoneId}, + {"url", "/api/v1/servers/localhost/zones/" + zoneId}, + {"name", val.first.toString()}, + {"kind", zone.d_servers.empty() ? "Native" : "Forwarded"}, + {"servers", servers}, + {"recursion_desired", zone.d_servers.empty() ? false : zone.d_rdForward}}); } resp->setJsonBody(doc); } @@ -309,35 +327,38 @@ static void apiServerZoneDetail(HttpRequest* req, HttpResponse* resp) SyncRes::domainmap_t::const_iterator iter = SyncRes::t_sstorage.domainmap->find(zonename); if (iter == SyncRes::t_sstorage.domainmap->end()) - throw ApiException("Could not find domain '"+zonename.toLogString()+"'"); + throw ApiException("Could not find domain '" + zonename.toLogString() + "'"); - if(req->method == "PUT") { + if (req->method == "PUT") { Json document = req->json(); doDeleteZone(zonename); doCreateZone(document); - reloadAuthAndForwards(); + reloadZoneConfiguration(); resp->body = ""; resp->status = 204; // No Content, but indicate success } - else if(req->method == "DELETE") { + else if (req->method == "DELETE") { if (!doDeleteZone(zonename)) { throw ApiException("Deleting domain failed"); } - reloadAuthAndForwards(); + reloadZoneConfiguration(); // empty body on success resp->body = ""; resp->status = 204; // No Content: declare that the zone is gone now - } else if(req->method == "GET") { + } + else if (req->method == "GET") { fillZone(zonename, resp); - } else { + } + else { throw HttpMethodNotAllowedException(); } } -static void apiServerSearchData(HttpRequest* req, HttpResponse* resp) { - if(req->method != "GET") +static void apiServerSearchData(HttpRequest* req, HttpResponse* resp) +{ + if (req->method != "GET") throw HttpMethodNotAllowedException(); string q = req->getvars["q"]; @@ -345,15 +366,14 @@ static void apiServerSearchData(HttpRequest* req, HttpResponse* resp) { throw ApiException("Query q can't be blank"); Json::array doc; - for(const SyncRes::domainmap_t::value_type& val : *SyncRes::t_sstorage.domainmap) { + for (const SyncRes::domainmap_t::value_type& val : *SyncRes::t_sstorage.domainmap) { string zoneId = apiZoneNameToId(val.first); string zoneName = val.first.toString(); if (pdns_ci_find(zoneName, q) != string::npos) { - doc.push_back(Json::object { - { "type", "zone" }, - { "zone_id", zoneId }, - { "name", zoneName } - }); + doc.push_back(Json::object{ + {"type", "zone"}, + {"zone_id", zoneId}, + {"name", zoneName}}); } // if zone name is an exact match, don't bother with returning all records/comments in it @@ -363,24 +383,24 @@ static void apiServerSearchData(HttpRequest* req, HttpResponse* resp) { const SyncRes::AuthDomain& zone = val.second; - for(const SyncRes::AuthDomain::records_t::value_type& rr : zone.d_records) { + for (const SyncRes::AuthDomain::records_t::value_type& rr : zone.d_records) { if (pdns_ci_find(rr.d_name.toString(), q) == string::npos && pdns_ci_find(rr.d_content->getZoneRepresentation(), q) == string::npos) continue; - doc.push_back(Json::object { - { "type", "record" }, - { "zone_id", zoneId }, - { "zone_name", zoneName }, - { "name", rr.d_name.toString() }, - { "content", rr.d_content->getZoneRepresentation() } - }); + doc.push_back(Json::object{ + {"type", "record"}, + {"zone_id", zoneId}, + {"zone_name", zoneName}, + {"name", rr.d_name.toString()}, + {"content", rr.d_content->getZoneRepresentation()}}); } } resp->setJsonBody(doc); } -static void apiServerCacheFlush(HttpRequest* req, HttpResponse* resp) { - if(req->method != "PUT") +static void apiServerCacheFlush(HttpRequest* req, HttpResponse* resp) +{ + if (req->method != "PUT") throw HttpMethodNotAllowedException(); DNSName canon = apiNameToDNSName(req->getvars["domain"]); @@ -390,17 +410,15 @@ static void apiServerCacheFlush(HttpRequest* req, HttpResponse* resp) { qtype = QType::chartocode(req->getvars["type"].c_str()); } - int count = g_recCache->doWipeCache(canon, subtree, qtype); - count += broadcastAccFunction([=]{return pleaseWipePacketCache(canon, subtree, qtype);}); - count += g_negCache->wipe(canon, subtree); - resp->setJsonBody(Json::object { - { "count", count }, - { "result", "Flushed cache." } - }); + struct WipeCacheResult res = wipeCaches(canon, subtree, qtype); + resp->setJsonBody(Json::object{ + {"count", res.record_count + res.packet_count + res.negative_record_count}, + {"result", "Flushed cache."}}); } -static void apiServerRPZStats(HttpRequest* req, HttpResponse* resp) { - if(req->method != "GET") +static void apiServerRPZStats(HttpRequest* req, HttpResponse* resp) +{ + if (req->method != "GET") throw HttpMethodNotAllowedException(); auto luaconf = g_luaconfs.getLocal(); @@ -408,7 +426,7 @@ static void apiServerRPZStats(HttpRequest* req, HttpResponse* resp) { Json::object ret; - for (size_t i=0; i < numZones; i++) { + for (size_t i = 0; i < numZones; i++) { auto zone = luaconf->dfe.getZone(i); if (zone == nullptr) continue; @@ -429,55 +447,61 @@ static void apiServerRPZStats(HttpRequest* req, HttpResponse* resp) { resp->setJsonBody(ret); } +static void prometheusMetrics(HttpRequest* req, HttpResponse* resp) +{ + static MetricDefinitionStorage s_metricDefinitions; -static void prometheusMetrics(HttpRequest *req, HttpResponse *resp) { - static MetricDefinitionStorage s_metricDefinitions; - - if (req->method != "GET") - throw HttpMethodNotAllowedException(); - - registerAllStats(); - - std::ostringstream output; - - // Argument controls disabling of any stats. So - // stats-api-disabled-list will be used to block returned stats. - auto varmap = getAllStatsMap(StatComponent::API); - for (const auto& tup : varmap) { - std::string metricName = tup.first; - std::string prometheusMetricName = tup.second.d_prometheusName; - std::string helpname = tup.second.d_prometheusName; - MetricDefinition metricDetails; - - if (s_metricDefinitions.getMetricDetails(metricName, metricDetails)) { - std::string prometheusTypeName = s_metricDefinitions.getPrometheusStringMetricType( - metricDetails.d_prometheusType); - - if (prometheusTypeName.empty()) { - continue; - } - if (metricDetails.d_prometheusType == PrometheusMetricType::multicounter) { - helpname = prometheusMetricName.substr(0, prometheusMetricName.find('{')); - } - else if (metricDetails.d_prometheusType == PrometheusMetricType::histogram) { - // name is XXX_count, strip the _count part - helpname = prometheusMetricName.substr(0, prometheusMetricName.length() - 6); - } - output << "# TYPE " << helpname << " " << prometheusTypeName << "\n"; - output << "# HELP " << helpname << " " << metricDetails.d_description << "\n"; - } - output << prometheusMetricName << " " << tup.second.d_value << "\n"; - } + if (req->method != "GET") + throw HttpMethodNotAllowedException(); - output << "# HELP pdns_recursor_info " << "Info from pdns_recursor, value is always 1" << "\n"; - output << "# TYPE pdns_recursor_info " << "gauge" << "\n"; - output << "pdns_recursor_info{version=\"" << VERSION << "\"} " << "1" << "\n"; + registerAllStats(); - resp->body = output.str(); - resp->headers["Content-Type"] = "text/plain"; - resp->status = 200; -} + std::ostringstream output; + // Argument controls disabling of any stats. So + // stats-api-disabled-list will be used to block returned stats. + auto varmap = getAllStatsMap(StatComponent::API); + for (const auto& tup : varmap) { + std::string metricName = tup.first; + std::string prometheusMetricName = tup.second.d_prometheusName; + std::string helpname = tup.second.d_prometheusName; + MetricDefinition metricDetails; + + if (s_metricDefinitions.getMetricDetails(metricName, metricDetails)) { + std::string prometheusTypeName = s_metricDefinitions.getPrometheusStringMetricType( + metricDetails.d_prometheusType); + + if (prometheusTypeName.empty()) { + continue; + } + if (metricDetails.d_prometheusType == PrometheusMetricType::multicounter) { + helpname = prometheusMetricName.substr(0, prometheusMetricName.find('{')); + } + else if (metricDetails.d_prometheusType == PrometheusMetricType::histogram) { + helpname = prometheusMetricName.substr(0, prometheusMetricName.find('{')); + // name is XXX_count, strip the _count part + helpname = helpname.substr(0, helpname.length() - 6); + } + output << "# TYPE " << helpname << " " << prometheusTypeName << "\n"; + output << "# HELP " << helpname << " " << metricDetails.d_description << "\n"; + } + output << prometheusMetricName << " " << tup.second.d_value << "\n"; + } + + output << "# HELP pdns_recursor_info " + << "Info from pdns_recursor, value is always 1" + << "\n"; + output << "# TYPE pdns_recursor_info " + << "gauge" + << "\n"; + output << "pdns_recursor_info{version=\"" << VERSION << "\"} " + << "1" + << "\n"; + + resp->body = output.str(); + resp->headers["Content-Type"] = "text/plain"; + resp->status = 200; +} #include "htmlfiles.h" @@ -485,17 +509,17 @@ static void serveStuff(HttpRequest* req, HttpResponse* resp) { resp->headers["Cache-Control"] = "max-age=86400"; - if(req->url.path == "/") + if (req->url.path == "/") req->url.path = "/index.html"; const string charset = "; charset=utf-8"; - if(boost::ends_with(req->url.path, ".html")) + if (boost::ends_with(req->url.path, ".html")) resp->headers["Content-Type"] = "text/html" + charset; - else if(boost::ends_with(req->url.path, ".css")) + else if (boost::ends_with(req->url.path, ".css")) resp->headers["Content-Type"] = "text/css" + charset; - else if(boost::ends_with(req->url.path,".js")) + else if (boost::ends_with(req->url.path, ".js")) resp->headers["Content-Type"] = "application/javascript" + charset; - else if(boost::ends_with(req->url.path, ".png")) + else if (boost::ends_with(req->url.path, ".png")) resp->headers["Content-Type"] = "image/png"; resp->headers["X-Content-Type-Options"] = "nosniff"; @@ -505,10 +529,11 @@ static void serveStuff(HttpRequest* req, HttpResponse* resp) resp->headers["X-XSS-Protection"] = "1; mode=block"; // resp->headers["Content-Security-Policy"] = "default-src 'self'; style-src 'self' 'unsafe-inline'"; - if (!req->url.path.empty() && g_urlmap.count(req->url.path.c_str()+1)) { - resp->body = g_urlmap.at(req->url.path.c_str()+1); + if (!req->url.path.empty() && g_urlmap.count(req->url.path.c_str() + 1)) { + resp->body = g_urlmap.at(req->url.path.c_str() + 1); resp->status = 200; - } else { + } + else { resp->status = 404; } } @@ -599,6 +624,9 @@ const std::map MetricDefinitionStorage::d_metrics MetricDefinition(PrometheusMetricType::multicounter, "Number of milliseconds spent in thread n")}, + {"zone-disallowed-notify", + MetricDefinition(PrometheusMetricType::counter, + "Number of NOTIFY operations denied because of allow-notify-for restrictions")}, {"dnssec-authentic-data-queries", MetricDefinition(PrometheusMetricType::counter, "Number of queries received with the AD bit set")}, @@ -825,6 +853,9 @@ const std::map MetricDefinitionStorage::d_metrics {"unauthorized-udp", MetricDefinition(PrometheusMetricType::counter, "Number of UDP questions denied because of allow-from restrictions")}, + {"source-disallowed-notify", + MetricDefinition(PrometheusMetricType::counter, + "Number of NOTIFY operations denied because of allow-notify-from restrictions")}, {"unexpected-packets", MetricDefinition(PrometheusMetricType::counter, "Number of answers from remote servers that were unexpected (might point to spoofing)")}, @@ -884,6 +915,24 @@ const std::map MetricDefinitionStorage::d_metrics {"udp-sndbuf-errors", MetricDefinition(PrometheusMetricType::counter, "From /proc/net/snmp SndbufErrors")}, + {"udp-in-csum-errors", + MetricDefinition(PrometheusMetricType::counter, + "From /proc/net/snmp InCsumErrors")}, + {"udp6-in-errors", + MetricDefinition(PrometheusMetricType::counter, + "From /proc/net/snmp6 InErrors")}, + {"udp6-noport-errors", + MetricDefinition(PrometheusMetricType::counter, + "From /proc/net/snmp6 NoPorts")}, + {"udp6-recvbuf-errors", + MetricDefinition(PrometheusMetricType::counter, + "From /proc/net/snmp6 RcvbufErrors")}, + {"udp6-sndbuf-errors", + MetricDefinition(PrometheusMetricType::counter, + "From /proc/net/snmp6 SndbufErrors")}, + {"udp6-in-csum-errors", + MetricDefinition(PrometheusMetricType::counter, + "From /proc/net/snmp6 InCsumErrors")}, {"cpu-iowait", MetricDefinition(PrometheusMetricType::counter, @@ -892,203 +941,203 @@ const std::map MetricDefinitionStorage::d_metrics MetricDefinition(PrometheusMetricType::counter, "Stolen time, which is the time spent by the whole system in other operating systems when running in a virtualized environment, in units of USER_HZ")}, - { "dnssec-result-bogus-invalid-denial", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid denial of existence proof could not be found")}, - - { "dnssec-result-bogus-invalid-dnskey-protocol", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because all DNSKEYs had invalid protocols")}, - - { "dnssec-result-bogus-missing-negative-indication", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a NODATA or NXDOMAIN answer lacked the required SOA and/or NSEC(3) records")}, - - { "dnssec-result-bogus-no-rrsig", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because required RRSIG records were not present in an answer")}, - - { "dnssec-result-bogus-no-valid-dnskey", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be found")}, - - { "dnssec-result-bogus-no-valid-rrsig", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because only invalid RRSIG records were present in an answer")}, - - { "dnssec-result-bogus-no-zone-key-bit-set", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because no DNSKEY with the Zone Key bit set was found")}, - - { "dnssec-result-bogus-revoked-dnskey", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because all DNSKEYs were revoked")}, - - { "dnssec-result-bogus-self-signed-ds", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a DS record was signed by itself")}, - - { "dnssec-result-bogus-signature-expired", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because the signature expired time in the RRSIG was in the past")}, - - { "dnssec-result-bogus-signature-not-yet-valid", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because the signature inception time in the RRSIG was not yet valid")}, - - { "dnssec-result-bogus-unable-to-get-dnskeys", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be retrieved")}, - - { "dnssec-result-bogus-unable-to-get-dss", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid DS could not be retrieved")}, - { "dnssec-result-bogus-unsupported-dnskey-algo", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a DNSKEY RRset contained only unsupported DNSSEC algorithms")}, - - { "dnssec-result-bogus-unsupported-ds-digest-type", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a DS RRset contained only unsupported digest types")}, - { "x-dnssec-result-bogus-invalid-denial", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid denial of existence proof could not be found")}, - - { "x-dnssec-result-bogus-invalid-dnskey-protocol", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because all DNSKEYs had invalid protocols")}, - - { "x-dnssec-result-bogus-missing-negative-indication", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a NODATA or NXDOMAIN answer lacked the required SOA and/or NSEC(3) records")}, - - { "x-dnssec-result-bogus-no-rrsig", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because required RRSIG records were not present in an answer")}, - - { "x-dnssec-result-bogus-no-valid-dnskey", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be found")}, - - { "x-dnssec-result-bogus-no-valid-rrsig", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because only invalid RRSIG records were present in an answer")}, - - { "x-dnssec-result-bogus-no-zone-key-bit-set", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because no DNSKEY with the Zone Key bit set was found")}, - - { "x-dnssec-result-bogus-revoked-dnskey", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because all DNSKEYs were revoked")}, - - { "x-dnssec-result-bogus-self-signed-ds", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a DS record was signed by itself")}, - - { "x-dnssec-result-bogus-signature-expired", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because the signature expired time in the RRSIG was in the past")}, - - { "x-dnssec-result-bogus-signature-not-yet-valid", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because the signature inception time in the RRSIG was not yet valid")}, - - { "x-dnssec-result-bogus-unable-to-get-dnskeys", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be retrieved")}, - - { "x-dnssec-result-bogus-unable-to-get-dss", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a valid DS could not be retrieved")}, - { "x-dnssec-result-bogus-unsupported-dnskey-algo", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a DNSKEY RRset contained only unsupported DNSSEC algorithms")}, - - { "x-dnssec-result-bogus-unsupported-ds-digest-type", - MetricDefinition(PrometheusMetricType::counter, - "number of DNSSEC validations that had the Bogus state because a DS RRset contained only unsupported digest types")}, - - { "proxy-protocol-invalid", - MetricDefinition(PrometheusMetricType::counter, - "invalid proxy-protocol headers received")}, - - { "record-cache-acquired", - MetricDefinition(PrometheusMetricType::counter, - "number of record cache lock acquisitions")}, - - { "record-cache-contended", - MetricDefinition(PrometheusMetricType::counter, - "number of contented record cache lock acquisitions")}, - - { "taskqueue-expired", - MetricDefinition(PrometheusMetricType::counter, - "number of tasks expired before they could be run")}, - - { "taskqueue-pushed", - MetricDefinition(PrometheusMetricType::counter, - "number of tasks pushed to the taskqueues")}, - - { "taskqueue-size", - MetricDefinition(PrometheusMetricType::gauge, - "number of tasks currently in the taskqueue")}, - - { "dot-outqueries", - MetricDefinition(PrometheusMetricType::counter, - "Number of outgoing DoT queries since starting")}, - - { "dns64-prefix-answers", - MetricDefinition(PrometheusMetricType::counter, - "Number of AAAA and PTR generated by a matching dns64-prefix")}, - { "aggressive-nsec-cache-entries", - MetricDefinition(PrometheusMetricType::counter, - "Number of entries in the aggressive NSEC cache")}, - - { "aggressive-nsec-cache-nsec-hits", - MetricDefinition(PrometheusMetricType::counter, - "Number of NSEC-related hits from the aggressive NSEC cache")}, - - { "aggressive-nsec-cache-nsec-wc-hits", - MetricDefinition(PrometheusMetricType::counter, - "Number of answers synthesized from the NSEC aggressive cache")}, - - { "aggressive-nsec-cache-nsec3-hits", - MetricDefinition(PrometheusMetricType::counter, - "Number of NSEC3-related hits from the aggressive NSEC cache")}, - - { "aggressive-nsec-cache-nsec3-wc-hits", - MetricDefinition(PrometheusMetricType::counter, - "Number of answers synthesized from the NSEC3 aggressive cache")}, + {"dnssec-result-bogus-invalid-denial", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid denial of existence proof could not be found")}, + + {"dnssec-result-bogus-invalid-dnskey-protocol", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because all DNSKEYs had invalid protocols")}, + + {"dnssec-result-bogus-missing-negative-indication", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a NODATA or NXDOMAIN answer lacked the required SOA and/or NSEC(3) records")}, + + {"dnssec-result-bogus-no-rrsig", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because required RRSIG records were not present in an answer")}, + + {"dnssec-result-bogus-no-valid-dnskey", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be found")}, + + {"dnssec-result-bogus-no-valid-rrsig", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because only invalid RRSIG records were present in an answer")}, + + {"dnssec-result-bogus-no-zone-key-bit-set", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because no DNSKEY with the Zone Key bit set was found")}, + + {"dnssec-result-bogus-revoked-dnskey", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because all DNSKEYs were revoked")}, + + {"dnssec-result-bogus-self-signed-ds", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a DS record was signed by itself")}, + + {"dnssec-result-bogus-signature-expired", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because the signature expired time in the RRSIG was in the past")}, + + {"dnssec-result-bogus-signature-not-yet-valid", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because the signature inception time in the RRSIG was not yet valid")}, + + {"dnssec-result-bogus-unable-to-get-dnskeys", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be retrieved")}, + + {"dnssec-result-bogus-unable-to-get-dss", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid DS could not be retrieved")}, + {"dnssec-result-bogus-unsupported-dnskey-algo", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a DNSKEY RRset contained only unsupported DNSSEC algorithms")}, + + {"dnssec-result-bogus-unsupported-ds-digest-type", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a DS RRset contained only unsupported digest types")}, + {"x-dnssec-result-bogus-invalid-denial", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid denial of existence proof could not be found")}, + + {"x-dnssec-result-bogus-invalid-dnskey-protocol", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because all DNSKEYs had invalid protocols")}, + + {"x-dnssec-result-bogus-missing-negative-indication", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a NODATA or NXDOMAIN answer lacked the required SOA and/or NSEC(3) records")}, + + {"x-dnssec-result-bogus-no-rrsig", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because required RRSIG records were not present in an answer")}, + + {"x-dnssec-result-bogus-no-valid-dnskey", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be found")}, + + {"x-dnssec-result-bogus-no-valid-rrsig", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because only invalid RRSIG records were present in an answer")}, + + {"x-dnssec-result-bogus-no-zone-key-bit-set", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because no DNSKEY with the Zone Key bit set was found")}, + + {"x-dnssec-result-bogus-revoked-dnskey", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because all DNSKEYs were revoked")}, + + {"x-dnssec-result-bogus-self-signed-ds", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a DS record was signed by itself")}, + + {"x-dnssec-result-bogus-signature-expired", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because the signature expired time in the RRSIG was in the past")}, + + {"x-dnssec-result-bogus-signature-not-yet-valid", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because the signature inception time in the RRSIG was not yet valid")}, + + {"x-dnssec-result-bogus-unable-to-get-dnskeys", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid DNSKEY could not be retrieved")}, + + {"x-dnssec-result-bogus-unable-to-get-dss", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a valid DS could not be retrieved")}, + {"x-dnssec-result-bogus-unsupported-dnskey-algo", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a DNSKEY RRset contained only unsupported DNSSEC algorithms")}, + + {"x-dnssec-result-bogus-unsupported-ds-digest-type", + MetricDefinition(PrometheusMetricType::counter, + "number of DNSSEC validations that had the Bogus state because a DS RRset contained only unsupported digest types")}, + + {"proxy-protocol-invalid", + MetricDefinition(PrometheusMetricType::counter, + "invalid proxy-protocol headers received")}, + + {"record-cache-acquired", + MetricDefinition(PrometheusMetricType::counter, + "number of record cache lock acquisitions")}, + + {"record-cache-contended", + MetricDefinition(PrometheusMetricType::counter, + "number of contented record cache lock acquisitions")}, + + {"taskqueue-expired", + MetricDefinition(PrometheusMetricType::counter, + "number of tasks expired before they could be run")}, + + {"taskqueue-pushed", + MetricDefinition(PrometheusMetricType::counter, + "number of tasks pushed to the taskqueues")}, + + {"taskqueue-size", + MetricDefinition(PrometheusMetricType::gauge, + "number of tasks currently in the taskqueue")}, + + {"dot-outqueries", + MetricDefinition(PrometheusMetricType::counter, + "Number of outgoing DoT queries since starting")}, + + {"dns64-prefix-answers", + MetricDefinition(PrometheusMetricType::counter, + "Number of AAAA and PTR generated by a matching dns64-prefix")}, + {"aggressive-nsec-cache-entries", + MetricDefinition(PrometheusMetricType::counter, + "Number of entries in the aggressive NSEC cache")}, + + {"aggressive-nsec-cache-nsec-hits", + MetricDefinition(PrometheusMetricType::counter, + "Number of NSEC-related hits from the aggressive NSEC cache")}, + + {"aggressive-nsec-cache-nsec-wc-hits", + MetricDefinition(PrometheusMetricType::counter, + "Number of answers synthesized from the NSEC aggressive cache")}, + + {"aggressive-nsec-cache-nsec3-hits", + MetricDefinition(PrometheusMetricType::counter, + "Number of NSEC3-related hits from the aggressive NSEC cache")}, + + {"aggressive-nsec-cache-nsec3-wc-hits", + MetricDefinition(PrometheusMetricType::counter, + "Number of answers synthesized from the NSEC3 aggressive cache")}, // For cumulative histogram, state the xxx_count name where xxx matches the name in rec_channel_rec - { "cumul-answers-count", - MetricDefinition(PrometheusMetricType::histogram, - "histogram of our answer times")}, - // For cumulative histogram, state the xxx_count name where xxx matches the name in rec_channel_rec - { "cumul-auth4answers-count", - MetricDefinition(PrometheusMetricType::histogram, - "histogram of authoritative answer times over IPv4")}, + {"cumul-clientanswers-count", + MetricDefinition(PrometheusMetricType::histogram, + "histogram of our answer times to clients")}, // For cumulative histogram, state the xxx_count name where xxx matches the name in rec_channel_rec - { "cumul-auth6answers-count", - MetricDefinition(PrometheusMetricType::histogram, - "histogram of authoritative answer times over IPV6")}, - - { "almost-expired-pushed", - MetricDefinition(PrometheusMetricType::counter, - "number of almost-expired tasks pushed")}, + {"cumul-authanswers-count4", + MetricDefinition(PrometheusMetricType::histogram, + "histogram of answer times of authoritative servers")}, + {"almost-expired-pushed", + MetricDefinition(PrometheusMetricType::counter, + "number of almost-expired tasks pushed")}, - { "almost-expired-run", - MetricDefinition(PrometheusMetricType::counter, - "number of almost-expired tasks run to completion")}, + {"almost-expired-run", + MetricDefinition(PrometheusMetricType::counter, + "number of almost-expired tasks run to completion")}, - { "almost-expired-exceptions", - MetricDefinition(PrometheusMetricType::counter, - "number of almost-expired tasks that caused an exception")}, + {"almost-expired-exceptions", + MetricDefinition(PrometheusMetricType::counter, + "number of almost-expired tasks that caused an exception")}, // For multicounters, state the first - { "policy-hits", - MetricDefinition(PrometheusMetricType::multicounter, - "Number of filter or RPZ policy hits")}, + {"policy-hits", + MetricDefinition(PrometheusMetricType::multicounter, + "Number of filter or RPZ policy hits")}, + + {"idle-tcpout-connections", + MetricDefinition(PrometheusMetricType::gauge, + "Number of connections in the TCP idle outgoing connections pool")}, + }; #define CHECK_PROMETHEUS_METRICS 0 @@ -1110,7 +1159,7 @@ static void validatePrometheusMetrics() MetricDefinition metricDetails; if (!s_metricDefinitions.getMetricDetails(metricName, metricDetails)) { - g_log<(fdm, arg()["webserver-address"], arg().asNum("webserver-port")); - d_ws->setApiKey(arg()["api-key"]); - d_ws->setPassword(arg()["webserver-password"]); + d_ws->setApiKey(arg()["api-key"], arg().mustDo("webserver-hash-plaintext-credentials")); + d_ws->setPassword(arg()["webserver-password"], arg().mustDo("webserver-hash-plaintext-credentials")); d_ws->setLogLevel(arg()["webserver-loglevel"]); NetmaskGroup acl; @@ -1137,21 +1186,24 @@ RecursorWebServer::RecursorWebServer(FDMultiplexer* fdm) d_ws->bind(); // legacy dispatch - d_ws->registerApiHandler("/jsonstat", [this](HttpRequest* req, HttpResponse* resp){jsonstat(req, resp);}, true); - d_ws->registerApiHandler("/api/v1/servers/localhost/cache/flush", &apiServerCacheFlush); - d_ws->registerApiHandler("/api/v1/servers/localhost/config/allow-from", &apiServerConfigAllowFrom); - d_ws->registerApiHandler("/api/v1/servers/localhost/config", &apiServerConfig); - d_ws->registerApiHandler("/api/v1/servers/localhost/rpzstatistics", &apiServerRPZStats); - d_ws->registerApiHandler("/api/v1/servers/localhost/search-data", &apiServerSearchData); - d_ws->registerApiHandler("/api/v1/servers/localhost/statistics", &apiServerStatistics, true); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones/", &apiServerZoneDetail); - d_ws->registerApiHandler("/api/v1/servers/localhost/zones", &apiServerZones); - d_ws->registerApiHandler("/api/v1/servers/localhost", &apiServerDetail, true); - d_ws->registerApiHandler("/api/v1/servers", &apiServer); - d_ws->registerApiHandler("/api", &apiDiscovery); + d_ws->registerApiHandler( + "/jsonstat", [this](HttpRequest* req, HttpResponse* resp) { jsonstat(req, resp); }, true); + d_ws->registerApiHandler("/api/v1/servers/localhost/cache/flush", apiServerCacheFlush); + d_ws->registerApiHandler("/api/v1/servers/localhost/config/allow-from", apiServerConfigAllowFrom); + d_ws->registerApiHandler("/api/v1/servers/localhost/config/allow-notify-from", &apiServerConfigAllowNotifyFrom); + d_ws->registerApiHandler("/api/v1/servers/localhost/config", apiServerConfig); + d_ws->registerApiHandler("/api/v1/servers/localhost/rpzstatistics", apiServerRPZStats); + d_ws->registerApiHandler("/api/v1/servers/localhost/search-data", apiServerSearchData); + d_ws->registerApiHandler("/api/v1/servers/localhost/statistics", apiServerStatistics, true); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones/", apiServerZoneDetail); + d_ws->registerApiHandler("/api/v1/servers/localhost/zones", apiServerZones); + d_ws->registerApiHandler("/api/v1/servers/localhost", apiServerDetail, true); + d_ws->registerApiHandler("/api/v1/servers", apiServer); + d_ws->registerApiHandler("/api/v1", apiDiscoveryV1); + d_ws->registerApiHandler("/api", apiDiscovery); for (const auto& u : g_urlmap) { - d_ws->registerWebHandler("/"+u.first, serveStuff); + d_ws->registerWebHandler("/" + u.first, serveStuff); } d_ws->registerWebHandler("/", serveStuff); @@ -1159,80 +1211,78 @@ RecursorWebServer::RecursorWebServer(FDMultiplexer* fdm) d_ws->go(); } -void RecursorWebServer::jsonstat(HttpRequest* req, HttpResponse *resp) +void RecursorWebServer::jsonstat(HttpRequest* req, HttpResponse* resp) { string command; - if(req->getvars.count("command")) { + if (req->getvars.count("command")) { command = req->getvars["command"]; req->getvars.erase("command"); } map stats; - if(command == "get-query-ring") { - typedef pair query_t; + if (command == "get-query-ring") { + typedef pair query_t; vector queries; - bool filter=!req->getvars["public-filtered"].empty(); + bool filter = !req->getvars["public-filtered"].empty(); - if(req->getvars["name"]=="servfail-queries") - queries=broadcastAccFunction >(pleaseGetServfailQueryRing); - else if(req->getvars["name"]=="bogus-queries") - queries=broadcastAccFunction >(pleaseGetBogusQueryRing); - else if(req->getvars["name"]=="queries") - queries=broadcastAccFunction >(pleaseGetQueryRing); + if (req->getvars["name"] == "servfail-queries") + queries = broadcastAccFunction>(pleaseGetServfailQueryRing); + else if (req->getvars["name"] == "bogus-queries") + queries = broadcastAccFunction>(pleaseGetBogusQueryRing); + else if (req->getvars["name"] == "queries") + queries = broadcastAccFunction>(pleaseGetQueryRing); - typedef map counts_t; + typedef map counts_t; counts_t counts; - unsigned int total=0; - for(const query_t& q : queries) { + unsigned int total = 0; + for (const query_t& q : queries) { total++; - if(filter) - counts[make_pair(getRegisteredName(q.first), q.second)]++; + if (filter) + counts[pair(getRegisteredName(q.first), q.second)]++; else - counts[make_pair(q.first, q.second)]++; + counts[pair(q.first, q.second)]++; } typedef std::multimap rcounts_t; rcounts_t rcounts; - for(counts_t::const_iterator i=counts.begin(); i != counts.end(); ++i) - rcounts.insert(make_pair(-i->second, i->first)); + for (counts_t::const_iterator i = counts.begin(); i != counts.end(); ++i) + rcounts.emplace(-i->second, i->first); Json::array entries; - unsigned int tot=0, totIncluded=0; - for(const rcounts_t::value_type& q : rcounts) { - totIncluded-=q.first; - entries.push_back(Json::array { - -q.first, q.second.first.toLogString(), DNSRecordContent::NumberToType(q.second.second) - }); - if(tot++>=100) - break; + unsigned int tot = 0, totIncluded = 0; + for (const rcounts_t::value_type& q : rcounts) { + totIncluded -= q.first; + entries.push_back(Json::array{ + -q.first, q.second.first.toLogString(), DNSRecordContent::NumberToType(q.second.second)}); + if (tot++ >= 100) + break; } - if(queries.size() != totIncluded) { - entries.push_back(Json::array { - (int)(queries.size() - totIncluded), "", "" - }); + if (queries.size() != totIncluded) { + entries.push_back(Json::array{ + (int)(queries.size() - totIncluded), "", ""}); } - resp->setJsonBody(Json::object { { "entries", entries } }); + resp->setJsonBody(Json::object{{"entries", entries}}); return; } - else if(command == "get-remote-ring") { + else if (command == "get-remote-ring") { vector queries; - if(req->getvars["name"]=="remotes") - queries=broadcastAccFunction >(pleaseGetRemotes); - else if(req->getvars["name"]=="servfail-remotes") - queries=broadcastAccFunction >(pleaseGetServfailRemotes); - else if(req->getvars["name"]=="bogus-remotes") - queries=broadcastAccFunction >(pleaseGetBogusRemotes); - else if(req->getvars["name"]=="large-answer-remotes") - queries=broadcastAccFunction >(pleaseGetLargeAnswerRemotes); - else if(req->getvars["name"]=="timeouts") - queries=broadcastAccFunction >(pleaseGetTimeouts); - - typedef map counts_t; + if (req->getvars["name"] == "remotes") + queries = broadcastAccFunction>(pleaseGetRemotes); + else if (req->getvars["name"] == "servfail-remotes") + queries = broadcastAccFunction>(pleaseGetServfailRemotes); + else if (req->getvars["name"] == "bogus-remotes") + queries = broadcastAccFunction>(pleaseGetBogusRemotes); + else if (req->getvars["name"] == "large-answer-remotes") + queries = broadcastAccFunction>(pleaseGetLargeAnswerRemotes); + else if (req->getvars["name"] == "timeouts") + queries = broadcastAccFunction>(pleaseGetTimeouts); + + typedef map counts_t; counts_t counts; - unsigned int total=0; - for(const ComboAddress& q : queries) { + unsigned int total = 0; + for (const ComboAddress& q : queries) { total++; counts[q]++; } @@ -1240,58 +1290,57 @@ void RecursorWebServer::jsonstat(HttpRequest* req, HttpResponse *resp) typedef std::multimap rcounts_t; rcounts_t rcounts; - for(counts_t::const_iterator i=counts.begin(); i != counts.end(); ++i) - rcounts.insert(make_pair(-i->second, i->first)); + for (counts_t::const_iterator i = counts.begin(); i != counts.end(); ++i) + rcounts.emplace(-i->second, i->first); Json::array entries; - unsigned int tot=0, totIncluded=0; - for(const rcounts_t::value_type& q : rcounts) { - totIncluded-=q.first; - entries.push_back(Json::array { - -q.first, q.second.toString() - }); - if(tot++>=100) - break; + unsigned int tot = 0, totIncluded = 0; + for (const rcounts_t::value_type& q : rcounts) { + totIncluded -= q.first; + entries.push_back(Json::array{ + -q.first, q.second.toString()}); + if (tot++ >= 100) + break; } - if(queries.size() != totIncluded) { - entries.push_back(Json::array { - (int)(queries.size() - totIncluded), "" - }); + if (queries.size() != totIncluded) { + entries.push_back(Json::array{ + (int)(queries.size() - totIncluded), ""}); } - resp->setJsonBody(Json::object { { "entries", entries } }); + resp->setJsonBody(Json::object{{"entries", entries}}); return; - } else { - resp->setErrorResult("Command '"+command+"' not found", 404); + } + else { + resp->setErrorResult("Command '" + command + "' not found", 404); } } +void AsyncServerNewConnectionMT(void* p) +{ + AsyncServer* server = (AsyncServer*)p; -void AsyncServerNewConnectionMT(void *p) { - AsyncServer *server = (AsyncServer*)p; - try { auto socket = server->accept(); // this is actually a shared_ptr if (socket) { server->d_asyncNewConnectionCallback(socket); } - } catch (NetworkError &e) { + } + catch (NetworkError& e) { // we're running in a shared process/thread, so can't just terminate/abort. - g_log<addReadFD(d_server_socket.getHandle(), [this] (int, boost::any&){ newConnection();}); + fdm->addReadFD(d_server_socket.getHandle(), [this](int, boost::any&) { newConnection(); }); } void AsyncServer::newConnection() @@ -1300,7 +1349,8 @@ void AsyncServer::newConnection() } // This is an entry point from FDM, so it needs to catch everything. -void AsyncWebServer::serveConnection(std::shared_ptr client) const { +void AsyncWebServer::serveConnection(std::shared_ptr client) const +{ if (!client->acl(d_acl)) { return; } @@ -1317,26 +1367,31 @@ void AsyncWebServer::serveConnection(std::shared_ptr client) const { yarl.initialize(&req); client->setNonBlocking(); - const struct timeval timeout{g_networkTimeoutMsec / 1000, static_cast(g_networkTimeoutMsec) % 1000 * 1000}; + const struct timeval timeout + { + g_networkTimeoutMsec / 1000, static_cast(g_networkTimeoutMsec) % 1000 * 1000 + }; std::shared_ptr tlsCtx{nullptr}; auto handler = std::make_shared("", client->releaseHandle(), timeout, tlsCtx, time(nullptr)); PacketBuffer data; try { - while(!req.complete) { + while (!req.complete) { auto ret = arecvtcp(data, 16384, handler, true); if (ret == LWResult::Result::Success) { string str(reinterpret_cast(data.data()), data.size()); req.complete = yarl.feed(str); - } else { + } + else { // read error OR EOF break; } } yarl.finalize(); - } catch (YaHTTP::ParseError &e) { + } + catch (YaHTTP::ParseError& e) { // request stays incomplete - g_log<= WebServer::LogLevel::None) { @@ -1348,38 +1403,39 @@ void AsyncWebServer::serveConnection(std::shared_ptr client) const { WebServer::handleRequest(req, resp); ostringstream ss; resp.write(ss); - const string &s = ss.str(); + const string& s = ss.str(); reply.insert(reply.end(), s.cbegin(), s.cend()); logResponse(resp, remote, logprefix); // now send the reply if (asendtcp(reply, handler) != LWResult::Result::Success || reply.empty()) { - g_log<close(); // needed to signal "done" to client } - catch(PDNSException &e) { - g_log<= WebServer::LogLevel::Normal) { - g_log<(d_server); if (!server) return; - server->asyncWaitForConnections(d_fdm, [this](const std::shared_ptr& c){serveConnection(c);}); + server->asyncWaitForConnections(d_fdm, [this](const std::shared_ptr& c) { serveConnection(c); }); } diff --git a/pdns/ws-recursor.hh b/pdns/ws-recursor.hh index f09177222d57..0a1a7fde3a3e 100644 --- a/pdns/ws-recursor.hh +++ b/pdns/ws-recursor.hh @@ -20,7 +20,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #pragma once -#include +#include #include "namespaces.hh" #include "mplexer.hh" #include "webserver.hh" @@ -28,16 +28,18 @@ class HttpRequest; class HttpResponse; -class AsyncServer : public Server { +class AsyncServer : public Server +{ public: - AsyncServer(const string &localaddress, int port) : Server(localaddress, port) + AsyncServer(const string& localaddress, int port) : + Server(localaddress, port) { d_server_socket.setNonBlocking(); }; - friend void AsyncServerNewConnectionMT(void *p); + friend void AsyncServerNewConnectionMT(void* p); - typedef boost::function< void(std::shared_ptr) > newconnectioncb_t; + typedef boost::function)> newconnectioncb_t; void asyncWaitForConnections(FDMultiplexer* fdm, const newconnectioncb_t& callback); private: @@ -49,8 +51,8 @@ private: class AsyncWebServer : public WebServer { public: - AsyncWebServer(FDMultiplexer* fdm, const string &listenaddress, int port) : - WebServer(listenaddress, port), d_fdm(fdm) { }; + AsyncWebServer(FDMultiplexer* fdm, const string& listenaddress, int port) : + WebServer(listenaddress, port), d_fdm(fdm){}; void go(); private: @@ -58,7 +60,8 @@ private: void serveConnection(std::shared_ptr socket) const; protected: - virtual std::shared_ptr createServer() override { + virtual std::shared_ptr createServer() override + { return std::make_shared(d_listenaddress, d_port); }; }; @@ -67,7 +70,7 @@ class RecursorWebServer : public boost::noncopyable { public: explicit RecursorWebServer(FDMultiplexer* fdm); - void jsonstat(HttpRequest* req, HttpResponse *resp); + void jsonstat(HttpRequest* req, HttpResponse* resp); private: std::unique_ptr d_ws{nullptr}; diff --git a/pdns/zone2json.cc b/pdns/zone2json.cc index 9da32bf5d54f..f2e0b9d394f6 100644 --- a/pdns/zone2json.cc +++ b/pdns/zone2json.cc @@ -104,6 +104,7 @@ try ::arg().set("zone-name","Specify an $ORIGIN in case it is not present")=""; ::arg().set("named-conf","Bind 8/9 named.conf to parse")=""; ::arg().set("max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0"; + ::arg().set("max-include-depth", "Maximum level of nested $INCLUDE depth when loading a zone from a file")="20"; ::arg().setCmd("help","Provide a helpful message"); ::arg().setCmd("version","Print the version"); @@ -171,6 +172,7 @@ try Json::array recs; ZoneParserTNG zpt(i->filename, i->name, BP.getDirectory()); zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps")); + zpt.setMaxIncludes(::arg().asNum("max-include-depth")); DNSResourceRecord rr; obj["name"] = i->name.toString(); diff --git a/pdns/zone2ldap.cc b/pdns/zone2ldap.cc index b1f3434da671..28039bf48e37 100644 --- a/pdns/zone2ldap.cc +++ b/pdns/zone2ldap.cc @@ -249,6 +249,7 @@ int main( int argc, char* argv[] ) args.set( "domainid", "Domain ID of the first domain found (incremented afterwards)" ) = "1"; args.set( "metadata-dn", "DN under which to store the domain metadata" ) = ""; args.set( "max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0"; + args.set( "max-include-depth", "Maximum nested $INCLUDE depth when loading a zone from a file")="20"; args.parse( argc, argv ); @@ -318,6 +319,7 @@ int main( int argc, char* argv[] ) g_zonename = i.name; ZoneParserTNG zpt(i.filename, i.name, BP.getDirectory()); zpt.setMaxGenerateSteps(args.asNum("max-generate-steps")); + zpt.setMaxIncludes(args.asNum("max-include-depth")); DNSResourceRecord rr; while(zpt.get(rr)) { callback(g_domainid, rr.qname, rr.qtype.toString(), encode_non_ascii(rr.content), rr.ttl); diff --git a/pdns/zone2sql.cc b/pdns/zone2sql.cc index 505fd1dfd7f5..51f18b91c029 100644 --- a/pdns/zone2sql.cc +++ b/pdns/zone2sql.cc @@ -213,6 +213,7 @@ try ::arg().set("named-conf","Bind 8/9 named.conf to parse")=""; ::arg().set("max-generate-steps", "Maximum number of $GENERATE steps when loading a zone from a file")="0"; + ::arg().set("max-include-depth", "Maximum nested $INCLUDE depth when loading a zone from a file")="20"; ::arg().setCmd("help","Provide a helpful message"); ::arg().setCmd("version","Print the version"); @@ -295,6 +296,7 @@ try ZoneParserTNG zpt(domain.filename, domain.name, BP.getDirectory()); zpt.setMaxGenerateSteps(::arg().asNum("max-generate-steps")); + zpt.setMaxIncludes(::arg().asNum("max-include-depth")); DNSResourceRecord rr; bool seenSOA=false; string comment; diff --git a/pdns/zoneparser-tng.cc b/pdns/zoneparser-tng.cc index b7c020fae382..ee0741f862bc 100644 --- a/pdns/zoneparser-tng.cc +++ b/pdns/zoneparser-tng.cc @@ -36,8 +36,9 @@ #include #include #include +#include -static string g_INstr("IN"); +const static string g_INstr("IN"); ZoneParserTNG::ZoneParserTNG(const string& fname, DNSName zname, string reldir, bool upgradeContent): d_reldir(std::move(reldir)), d_zonename(std::move(zname)), d_defaultttl(3600), @@ -57,10 +58,34 @@ ZoneParserTNG::ZoneParserTNG(const vector& zonedata, DNSName zname, boo void ZoneParserTNG::stackFile(const std::string& fname) { - FILE *fp=fopen(fname.c_str(), "r"); - if(!fp) { - std::error_code ec (errno,std::generic_category()); - throw std::system_error(ec, "Unable to open file '"+fname+"': "+stringerror()); + if (d_filestates.size() >= d_maxIncludes) { + std::error_code ec (0, std::generic_category()); + throw std::system_error(ec, "Include limit reached"); + } + int fd = open(fname.c_str(), O_RDONLY, 0); + if (fd == -1) { + int err = errno; + std::error_code ec (err, std::generic_category()); + throw std::system_error(ec, "Unable to open file '" + fname + "': " + stringerror(err)); + } + struct stat st; + if (fstat(fd, &st) == -1) { + int err = errno; + close(fd); + std::error_code ec (err, std::generic_category()); + throw std::system_error(ec, "Unable to stat file '" + fname + "': " + stringerror(err)); + } + if (!S_ISREG(st.st_mode)) { + close(fd); + std::error_code ec (0, std::generic_category()); + throw std::system_error(ec, "File '" + fname + "': not a regular file"); + } + FILE *fp = fdopen(fd, "r"); + if (!fp) { + int err = errno; + close(fd); + std::error_code ec (err, std::generic_category()); + throw std::system_error(ec, "Unable to open file '" + fname + "': " + stringerror(err)); } filestate fs(fp, fname); @@ -139,8 +164,10 @@ unsigned int ZoneParserTNG::makeTTLFromZone(const string& str) bool ZoneParserTNG::getTemplateLine() { - if(d_templateparts.empty() || d_templatecounter > d_templatestop) // no template, or done with + if (d_templateparts.empty() || d_templateCounterWrapped || d_templatecounter > d_templatestop) { + // no template, or done with return false; + } string retline; for(parts_t::const_iterator iter = d_templateparts.begin() ; iter != d_templateparts.end(); ++iter) { @@ -198,8 +225,15 @@ bool ZoneParserTNG::getTemplateLine() if (extracted < 1) { throw PDNSException("Unable to parse offset, width and radix for $GENERATE's lhs from '"+spec+"' "+getLineOfFile()); } + if (width < 0) { + throw PDNSException("Invalid width ("+std::to_string(width)+") for $GENERATE's lhs from '"+spec+"' "+getLineOfFile()); + } char tmp[80]; + + /* a width larger than the output buffer does not make any sense */ + width = std::min(width, static_cast(sizeof(tmp))); + switch (radix) { case 'o': snprintf(tmp, sizeof(tmp), "%0*o", width, d_templatecounter + offset); @@ -222,7 +256,13 @@ bool ZoneParserTNG::getTemplateLine() } retline+=outpart; } - d_templatecounter+=d_templatestep; + + if ((d_templatestop - d_templatecounter) < d_templatestep) { + d_templateCounterWrapped = true; + } + else { + d_templatecounter += d_templatestep; + } d_line = retline; return true; @@ -333,31 +373,86 @@ bool ZoneParserTNG::get(DNSResourceRecord& rr, std::string* comment) // The range part can be one of two forms: start-stop or start-stop/step. If the first // form is used, then step is set to 1. start, stop and step must be positive // integers between 0 and (2^31)-1. start must not be larger than stop. - string range=makeString(d_line, d_parts[1]); - d_templatestep=1; - d_templatestop=0; - int extracted = sscanf(range.c_str(),"%" SCNu32 "-%" SCNu32 "/%" SCNu32, &d_templatecounter, &d_templatestop, &d_templatestep); - if (extracted == 2) { - d_templatestep=1; + // http://www.zytrax.com/books/dns/ch8/generate.html + string range = makeString(d_line, d_parts.at(1)); + + auto splitOnOnlyOneSeparator = [range](const std::string& input, std::vector& output, char separator) { + output.clear(); + + auto pos = input.find(separator); + if (pos == string::npos) { + output.emplace_back(input); + return; + } + if (pos == (input.size()-1)) { + /* ends on a separator!? */ + throw std::runtime_error("Invalid range from $GENERATE parameters '" + range + "'"); + } + auto next = input.find(separator, pos + 1); + if (next != string::npos) { + /* more than one separator */ + throw std::runtime_error("Invalid range from $GENERATE parameters '" + range + "'"); + } + output.emplace_back(input.substr(0, pos)); + output.emplace_back(input.substr(pos + 1)); + }; + + std::vector fields; + splitOnOnlyOneSeparator(range, fields, '-'); + if (fields.size() != 2) { + throw std::runtime_error("Invalid range from $GENERATE parameters '" + range + "'"); } - else if (extracted != 3) { - throw exception("Invalid range from $GENERATE parameters '" + range + "'"); + + auto parseValue = [](const std::string& parameters, const std::string& name, const std::string& str, uint32_t& value) { + try { + auto got = std::stoul(str); + if (got > std::numeric_limits::max()) { + throw std::runtime_error("Invalid " + name + " value in $GENERATE parameters '" + parameters + "'"); + } + value = static_cast(got); + } + catch (const std::exception& e) { + throw std::runtime_error("Invalid " + name + " value in $GENERATE parameters '" + parameters + "': " + e.what()); + } + }; + + parseValue(range, "start", fields.at(0), d_templatecounter); + + /* now the remaining part(s) */ + range = std::move(fields.at(1)); + splitOnOnlyOneSeparator(range, fields, '/'); + + if (fields.size() > 2) { + throw std::runtime_error("Invalid range from $GENERATE parameters '" + range + "'"); } + + parseValue(range, "stop", fields.at(0), d_templatestop); + + if (fields.size() == 2) { + parseValue(range, "step", fields.at(1), d_templatestep); + } + else { + d_templatestep = 1; + } + if (d_templatestep < 1 || d_templatestop < d_templatecounter) { - throw exception("Invalid $GENERATE parameters"); + throw std::runtime_error("Invalid $GENERATE parameters"); } if (d_maxGenerateSteps != 0) { size_t numberOfSteps = (d_templatestop - d_templatecounter) / d_templatestep; if (numberOfSteps > d_maxGenerateSteps) { - throw exception("The number of $GENERATE steps (" + std::to_string(numberOfSteps) + ") is too high, the maximum is set to " + std::to_string(d_maxGenerateSteps)); + throw std::runtime_error("The number of $GENERATE steps (" + std::to_string(numberOfSteps) + ") is too high, the maximum is set to " + std::to_string(d_maxGenerateSteps)); } } - d_templateline=d_line; + + d_templateline = d_line; d_parts.pop_front(); d_parts.pop_front(); - d_templateparts=d_parts; + d_templateparts = d_parts; + d_templateCounterWrapped = false; + goto retry; } else diff --git a/pdns/zoneparser-tng.hh b/pdns/zoneparser-tng.hh index a15f7630a645..1b7562766a27 100644 --- a/pdns/zoneparser-tng.hh +++ b/pdns/zoneparser-tng.hh @@ -49,6 +49,10 @@ public: { d_maxGenerateSteps = max; } + void setMaxIncludes(size_t max) + { + d_maxIncludes = max; + } private: bool getLine(); bool getTemplateLine(); @@ -73,10 +77,12 @@ private: std::stack d_filestates; parts_t d_templateparts; size_t d_maxGenerateSteps{0}; + size_t d_maxIncludes{20}; int d_defaultttl; uint32_t d_templatecounter, d_templatestop, d_templatestep; bool d_havedollarttl; bool d_fromfile; bool d_generateEnabled{true}; bool d_upgradeContent; + bool d_templateCounterWrapped{false}; }; diff --git a/regression-tests.api/requirements.txt b/regression-tests.api/requirements.txt index b97f1065b122..4e811cd87b36 100644 --- a/regression-tests.api/requirements.txt +++ b/regression-tests.api/requirements.txt @@ -1,5 +1,5 @@ -requests==2.20.0 -nose==1.3.0 +requests==2.31.0 +nose>=1.3.7 parameterized mysql-connector psycopg2 diff --git a/regression-tests.api/runtests.py b/regression-tests.api/runtests.py index 956047e99387..9b12cc628715 100755 --- a/regression-tests.api/runtests.py +++ b/regression-tests.api/runtests.py @@ -5,6 +5,7 @@ from __future__ import print_function import os import requests +from requests.exceptions import HTTPError import shutil import subprocess import sys @@ -87,6 +88,13 @@ ::1 """ +ACL_NOTIFY_LIST_TPL = """ +# Generated by runtests.py +# local host +127.0.0.1 +::1 +""" + REC_EXAMPLE_COM_CONF_TPL = """ # Generated by runtests.py auth-zones+=example.com=../regression-tests/zones/example.com @@ -98,6 +106,7 @@ forward-zones= forward-zones-recurse= allow-from-file=acl.list +allow-notify-from-file=acl-notify.list api-config-dir=%(conf_dir)s include-dir=%(conf_dir)s """ @@ -214,6 +223,8 @@ def run_check_call(cmd, *args, **kwargs): ensure_empty_dir(conf_dir) with open('acl.list', 'w') as acl_list: acl_list.write(ACL_LIST_TPL) + with open('acl-notify.list', 'w') as acl_notify_list: + acl_notify_list.write(ACL_NOTIFY_LIST_TPL) with open('recursor.conf', 'w') as recursor_conf: recursor_conf.write(REC_CONF_TPL % locals()) with open(conf_dir+'/example.com..conf', 'w') as conf_file: @@ -225,22 +236,30 @@ def run_check_call(cmd, *args, **kwargs): # Now run pdns and the tests. print("Launching server...") print(format_call_args(servercmd)) -serverproc = subprocess.Popen(servercmd, close_fds=True) +serverproc = subprocess.Popen(servercmd, close_fds=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) print("Waiting for webserver port to become available...") available = False +time.sleep(1) for try_number in range(0, 10): try: res = requests.get('http://127.0.0.1:%s/' % WEBPORT) available = True break - except: - time.sleep(0.5) + except HTTPError as http_err: + print(f'HTTP error occurred: {http_err}') + except Exception as err: + print(f'Other error occurred: {err}') + time.sleep(1) if not available: print("Webserver port not reachable after 10 tries, giving up.") serverproc.terminate() serverproc.wait() + print("==STDOUT===") + print(proc.stdout.read()) + print("==STDERRR===") + print(proc.stderr.read()) sys.exit(2) print("Query for example.com/A to create statistic data...") @@ -279,5 +298,9 @@ def run_check_call(cmd, *args, **kwargs): raw_input() serverproc.terminate() serverproc.wait() + print("==STDOUT===") + print(serverproc.stdout.read()) + print("==STDERRR===") + print(serverproc.stderr.read()) sys.exit(returncode) diff --git a/regression-tests.api/test_Discovery.py b/regression-tests.api/test_Discovery.py index e790f4b33aa1..930c8d56b28d 100644 --- a/regression-tests.api/test_Discovery.py +++ b/regression-tests.api/test_Discovery.py @@ -8,3 +8,8 @@ def test_discovery(self): self.assert_success_json(r) lst = r.json() self.assertEqual(lst, [{'version': 1, 'url': '/api/v1'}]) + + r = self.session.get(self.url("/api/v1")) + self.assert_success_json(r) + lst = r.json() + self.assertEqual(lst, [{'api_features': [], 'server_url': '/api/v1/servers{/server}'}]) diff --git a/regression-tests.api/test_RecursorConfig.py b/regression-tests.api/test_RecursorConfig.py index d8cba15138bf..774d366ad089 100644 --- a/regression-tests.api/test_RecursorConfig.py +++ b/regression-tests.api/test_RecursorConfig.py @@ -4,7 +4,7 @@ @unittest.skipIf(not is_recursor(), "Only applicable to recursors") -class RecursorConfig(ApiTestCase): +class RecursorAllowFromConfig(ApiTestCase): def test_config_allow_from_get(self): r = self.session.get(self.url("/api/v1/servers/localhost/config/allow-from")) @@ -32,3 +32,34 @@ def test_config_allow_from_replace_error(self): self.assertEqual(r.status_code, 422) data = r.json() self.assertIn('Unable to convert', data['error']) + + +@unittest.skipIf(not is_recursor(), "Only applicable to recursors") +class RecursorAllowNotifyFromConfig(ApiTestCase): + + def test_config_allow_notify_from_get(self): + r = self.session.get(self.url("/api/v1/servers/localhost/config/allow-notify-from")) + self.assert_success_json(r) + + def test_config_allow_notify_from_replace(self): + payload = {'value': ["127.0.0.1"]} + r = self.session.put( + self.url("/api/v1/servers/localhost/config/allow-notify-from"), + data=json.dumps(payload), + headers={'content-type': 'application/json'}) + self.assert_success_json(r) + data = r.json() + self.assertIn("value", data) + self.assertEqual(len(data["value"]), 1) + self.assertEqual("127.0.0.1/32", data["value"][0]) + + def test_config_allow_notify_from_replace_error(self): + """Test the error case, should return 422.""" + payload = {'value': ["abcdefgh"]} + r = self.session.put( + self.url("/api/v1/servers/localhost/config/allow-notify-from"), + data=json.dumps(payload), + headers={'content-type': 'application/json'}) + self.assertEqual(r.status_code, 422) + data = r.json() + self.assertIn('Unable to convert', data['error']) diff --git a/regression-tests.api/test_Zones.py b/regression-tests.api/test_Zones.py index ab5a7c119727..bd12e81bc83d 100644 --- a/regression-tests.api/test_Zones.py +++ b/regression-tests.api/test_Zones.py @@ -1409,14 +1409,14 @@ def test_rrset_other_and_exclusive(self, qtype): self.assertIn('Conflicts with pre-existing RRset', r.json()['error']) @parameterized.expand([ - ('SOA', ['ns1.example.org. test@example.org. 10 10800 3600 604800 3600', 'ns2.example.org. test@example.org. 10 10800 3600 604800 3600']), - ('CNAME', ['01.example.org.', '02.example.org.']), + ('', 'SOA', ['ns1.example.org. test@example.org. 10 10800 3600 604800 3600', 'ns2.example.org. test@example.org. 10 10800 3600 604800 3600']), + ('sub.', 'CNAME', ['01.example.org.', '02.example.org.']), ]) - def test_rrset_single_qtypes(self, qtype, contents): + def test_rrset_single_qtypes(self, label, qtype, contents): name, payload, zone = self.create_zone() rrset = { 'changetype': 'replace', - 'name': 'sub.'+name, + 'name': label + name, 'type': qtype, 'ttl': 3600, 'records': [ @@ -1468,6 +1468,101 @@ def test_rrset_zone_apex(self): headers={'content-type': 'application/json'}) self.assert_success(r) # user should be able to create DNAME at APEX as per RFC 6672 section 2.3 + @parameterized.expand([ + ('SOA', 'ns1.example.org. test@example.org. 10 10800 3600 604800 1800'), + ('DNSKEY', '257 3 8 AwEAAb/+pXOZWYQ8mv9WM5dFva8WU9jcIUdDuEjldbyfnkQ/xlrJC5zAEfhYhrea3SmIPmMTDimLqbh3/4SMTNPTUF+9+U1vpNfIRTFadqsmuU9Fddz3JqCcYwEpWbReg6DJOeyu+9oBoIQkPxFyLtIXEPGlQzrynKubn04Cx83I6NfzDTraJT3jLHKeW5PVc1ifqKzHz5TXdHHTA7NkJAa0sPcZCoNE1LpnJI/wcUpRUiuQhoLFeT1E432GuPuZ7y+agElGj0NnBxEgnHrhrnZWUbULpRa/il+Cr5Taj988HqX9Xdm6FjcP4Lbuds/44U7U8du224Q8jTrZ57Yvj4VDQKc='), + ]) + def test_only_at_apex(self, qtype, content): + name, payload, zone = self.create_zone(soa_edit_api='') + rrset = { + 'changetype': 'replace', + 'name': name, + 'type': qtype, + 'ttl': 3600, + 'records': [ + { + "content": content, + "disabled": False + }, + ] + } + payload = {'rrsets': [rrset]} + r = self.session.patch( + self.url("/api/v1/servers/localhost/zones/" + name), + data=json.dumps(payload), + headers={'content-type': 'application/json'}) + self.assert_success(r) + # verify that the new record is there + data = self.session.get(self.url("/api/v1/servers/localhost/zones/" + name)).json() + self.assertEqual(get_rrset(data, name, qtype)['records'], rrset['records']) + + rrset = { + 'changetype': 'replace', + 'name': 'sub.' + name, + 'type': qtype, + 'ttl': 3600, + 'records': [ + { + "content": content, + "disabled": False + }, + ] + } + payload = {'rrsets': [rrset]} + r = self.session.patch(self.url("/api/v1/servers/localhost/zones/" + name), data=json.dumps(payload), + headers={'content-type': 'application/json'}) + self.assertEqual(r.status_code, 422) + self.assertIn('only allowed at apex', r.json()['error']) + data = self.session.get(self.url("/api/v1/servers/localhost/zones/" + name)).json() + self.assertIsNone(get_rrset(data, 'sub.' + name, qtype)) + + @parameterized.expand([ + ('DS', '44030 8 2 d4c3d5552b8679faeebc317e5f048b614b2e5f607dc57f1553182d49ab2179f7'), + ]) + def test_not_allowed_at_apex(self, qtype, content): + name, payload, zone = self.create_zone() + rrset = { + 'changetype': 'replace', + 'name': name, + 'type': qtype, + 'ttl': 3600, + 'records': [ + { + "content": content, + "disabled": False + }, + ] + } + payload = {'rrsets': [rrset]} + r = self.session.patch(self.url("/api/v1/servers/localhost/zones/" + name), data=json.dumps(payload), + headers={'content-type': 'application/json'}) + self.assertEqual(r.status_code, 422) + self.assertIn('not allowed at apex', r.json()['error']) + data = self.session.get(self.url("/api/v1/servers/localhost/zones/" + name)).json() + self.assertIsNone(get_rrset(data, 'sub.' + name, qtype)) + + rrset = { + 'changetype': 'replace', + 'name': 'sub.' + name, + 'type': qtype, + 'ttl': 3600, + 'records': [ + { + "content": content, + "disabled": False + }, + ] + } + payload = {'rrsets': [rrset]} + r = self.session.patch( + self.url("/api/v1/servers/localhost/zones/" + name), + data=json.dumps(payload), + headers={'content-type': 'application/json'}) + self.assert_success(r) + # verify that the new record is there + data = self.session.get(self.url("/api/v1/servers/localhost/zones/" + name)).json() + self.assertEqual(get_rrset(data, 'sub.' + name, qtype)['records'], rrset['records']) + def test_rr_svcb(self): name, payload, zone = self.create_zone() rrset = { diff --git a/regression-tests.auth-py/authtests.py b/regression-tests.auth-py/authtests.py index f9bbe80b4349..2047e6d57498 100644 --- a/regression-tests.auth-py/authtests.py +++ b/regression-tests.auth-py/authtests.py @@ -547,8 +547,12 @@ def assertRcodeEqual(self, msg, rcode): raise TypeError("rcode is neither a str nor int") if msg.rcode() != rcode: - msgRcode = dns.rcode.to_text(msg.rcode()) - wantedRcode = dns.rcode.to_text(rcode) + try: + msgRcode = dns.rcode.to_text(msg.rcode()) + wantedRcode = dns.rcode.to_text(rcode) + except AttributeError: + msgRcode = msg.rcode() + wantedRcode = rcode raise AssertionError("Rcode for %s is %s, expected %s." % (msg.question[0].to_text(), msgRcode, wantedRcode)) diff --git a/regression-tests.auth-py/cookiesoption.py b/regression-tests.auth-py/cookiesoption.py new file mode 120000 index 000000000000..11e592d1d8b5 --- /dev/null +++ b/regression-tests.auth-py/cookiesoption.py @@ -0,0 +1 @@ +../regression-tests.dnsdist/cookiesoption.py \ No newline at end of file diff --git a/regression-tests.auth-py/proxyprotocol.py b/regression-tests.auth-py/proxyprotocol.py new file mode 120000 index 000000000000..2a3d79b0753e --- /dev/null +++ b/regression-tests.auth-py/proxyprotocol.py @@ -0,0 +1 @@ +../regression-tests.common/proxyprotocol.py \ No newline at end of file diff --git a/regression-tests.auth-py/requirements.txt b/regression-tests.auth-py/requirements.txt index 6f6ac0bab079..ee1ae3f24b65 100644 --- a/regression-tests.auth-py/requirements.txt +++ b/regression-tests.auth-py/requirements.txt @@ -1,4 +1,4 @@ -dnspython>=1.11 +dnspython==2.1.0 nose>=1.3.7 Twisted>0.15.0 requests>=2.18.4 diff --git a/regression-tests.auth-py/runtests b/regression-tests.auth-py/runtests index 496652c48507..064d0bc89fd9 100755 --- a/regression-tests.auth-py/runtests +++ b/regression-tests.auth-py/runtests @@ -16,6 +16,7 @@ mkdir -p configs export PDNS=${PDNS:-${PWD}/../pdns/pdns_server} export PDNSUTIL=${PDNSUTIL:-${PWD}/../pdns/pdnsutil} +export PDNSCONTROL=${PDNSCONTROL:-${PWD}/../pdns/pdns_control} export PREFIX=127.0.0 diff --git a/regression-tests.auth-py/test_Carbon.py b/regression-tests.auth-py/test_Carbon.py new file mode 100644 index 000000000000..6718b15ad5ca --- /dev/null +++ b/regression-tests.auth-py/test_Carbon.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +import threading +import socket +import sys +import time +import os +from queue import Queue + +from authtests import AuthTest + +class TestCarbon(AuthTest): + _carbonNamespace = 'NS' + _carbonInstance = 'Instance' + _carbonServerName = "carbonname1" + _carbonInterval = 2 + _carbonServer1Port = 8000 + _carbonServer2Port = 8001 + _carbonQueue1 = Queue() + _carbonQueue2 = Queue() + _carbonCounters = {} + _config_template = """ + launch=bind + carbon-namespace=%s + carbon-instance=%s + carbon-interval=%s + carbon-ourname=%s + carbon-server=127.0.0.1:%s,127.0.01:%s + """ % (_carbonNamespace, _carbonInstance, _carbonInterval, _carbonServerName, _carbonServer1Port, _carbonServer2Port) + + @classmethod + def CarbonResponder(cls, port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + try: + sock.bind(("127.0.0.1", port)) + except socket.error as e: + print("Error binding in the Carbon responder: %s" % str(e)) + sys.exit(1) + + sock.listen(100) + while True: + (conn, _) = sock.accept() + conn.settimeout(2.0) + lines = b'' + while True: + data = conn.recv(4096) + if not data: + break + lines += data + + if port == cls._carbonServer1Port: + cls._carbonQueue1.put(lines, True, timeout=2.0) + else: + cls._carbonQueue2.put(lines, True, timeout=2.0) + if threading.currentThread().name in cls._carbonCounters: + cls._carbonCounters[threading.currentThread().name] += 1 + else: + cls._carbonCounters[threading.currentThread().name] = 1 + + conn.close() + sock.close() + + @classmethod + def startResponders(cls): + cls._CarbonResponder1 = threading.Thread(name='Carbon Responder 1', target=cls.CarbonResponder, args=[cls._carbonServer1Port]) + cls._CarbonResponder1.setDaemon(True) + cls._CarbonResponder1.start() + + cls._CarbonResponder2 = threading.Thread(name='Carbon Responder 2', target=cls.CarbonResponder, args=[cls._carbonServer2Port]) + cls._CarbonResponder2.setDaemon(True) + cls._CarbonResponder2.start() + + def testCarbon(self): + """ + Carbon: send data to 2 carbon servers + """ + # wait for the carbon data to be sent + time.sleep(self._carbonInterval + 1) + + # check if the servers have received our data + # we will block for a short while if the data is not already there, + # and an exception will be raised after the timeout + # first server + data1 = self._carbonQueue1.get(block=True, timeout=2.0) + # second server + data2 = self._carbonQueue2.get(block=True, timeout=2.0) + after = time.time() + + self.assertTrue(data1) + self.assertTrue(len(data1.splitlines()) > 1) + expectedStart = b"%s.%s.%s." % (self._carbonNamespace.encode('UTF8'), self._carbonServerName.encode('UTF-8'), self._carbonInstance.encode('UTF8')) + for line in data1.splitlines(): + self.assertTrue(line.startswith(expectedStart)) + parts = line.split(b' ') + self.assertEqual(len(parts), 3) + self.assertTrue(parts[1].isdigit()) + self.assertTrue(parts[2].isdigit()) + self.assertTrue(int(parts[2]) <= int(after)) + + self.assertTrue(data2) + self.assertTrue(len(data2.splitlines()) > 1) + expectedStart = b"%s.%s.%s." % (self._carbonNamespace.encode('UTF8'), self._carbonServerName.encode('UTF-8'), self._carbonInstance.encode('UTF8')) + for line in data2.splitlines(): + self.assertTrue(line.startswith(expectedStart)) + parts = line.split(b' ') + self.assertEqual(len(parts), 3) + self.assertTrue(parts[1].isdigit()) + self.assertTrue(parts[2].isdigit()) + self.assertTrue(int(parts[2]) <= int(after)) + + # make sure every carbon server has received at least one connection + for key in self._carbonCounters: + value = self._carbonCounters[key] + self.assertTrue(value >= 1) + diff --git a/regression-tests.auth-py/test_Cookies.py b/regression-tests.auth-py/test_Cookies.py new file mode 100644 index 000000000000..ba67da897828 --- /dev/null +++ b/regression-tests.auth-py/test_Cookies.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +import dns + +from authtests import AuthTest + + +class TestEdnsCookies(AuthTest): + _config_template = """ +launch=bind +edns-cookie-secret=aabbccddeeff11223344556677889900 +""" + + _zones = { + 'example.org': """ +example.org. 3600 IN SOA {soa} +example.org. 3600 IN NS ns1.example.org. +example.org. 3600 IN NS ns2.example.org. +ns1.example.org. 3600 IN A 192.0.2.10 +ns2.example.org. 3600 IN A 192.0.2.11 + +www.example.org. 3600 IN A 192.0.2.5 + """, + } + + def sendAndExpectNoCookie(self, msg, rcode): + res = self.sendUDPQuery(msg) + self.assertRcodeEqual(res, rcode) + self.assertFalse(any([opt.otype == dns.edns.COOKIE for + opt in res.options])) + + def getCookieFromServer(self): + opts = [ + dns.edns.GenericOption(dns.edns.COOKIE, + b'\x22\x11\x33\x44\x55\x66\x77\x88')] + query = dns.message.make_query('www.example.org', 'A', options=opts) + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, 23) # BADCOOKIE + for opt in res.options: + if opt.otype == dns.edns.COOKIE: + return opt + self.fail() + + def testNoCookie(self): + query = dns.message.make_query('www.example.org', 'A', use_edns=0) + self.sendAndExpectNoCookie(query, dns.rcode.NOERROR) + + def testClientCookieTooShort(self): + opts = [dns.edns.GenericOption(dns.edns.COOKIE, b'\x22')] + query = dns.message.make_query('www.example.org', 'A', options=opts) + self.sendAndExpectNoCookie(query, dns.rcode.FORMERR) + + opts = [dns.edns.GenericOption(dns.edns.COOKIE, + b'\x22\x11\x33\x44\x55\x66\x77')] + query = dns.message.make_query('www.example.org', 'A', options=opts) + self.sendAndExpectNoCookie(query, dns.rcode.FORMERR) + + def testServerCookieTooShort(self): + opts = [ + dns.edns.GenericOption(dns.edns.COOKIE, + b'\x22\x11\x33\x44\x55\x66\x77\x88\x99')] + query = dns.message.make_query('www.example.org', 'A', options=opts) + self.sendAndExpectNoCookie(query, dns.rcode.FORMERR) + + opts = [ + dns.edns.GenericOption(dns.edns.COOKIE, + b'\x22\x11\x33\x44\x55\x66\x77\x88' + + b'\x22\x11\x33\x44\x55\x66\x77')] + query = dns.message.make_query('www.example.org', 'A', options=opts) + self.sendAndExpectNoCookie(query, dns.rcode.FORMERR) + + def testOnlyClientCookie(self): + opts = [ + dns.edns.GenericOption(dns.edns.COOKIE, + b'\x22\x11\x33\x44\x55\x66\x77\x88')] + query = dns.message.make_query('www.example.org', 'A', options=opts) + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, 23) # BADCOOKIE + self.assertTrue(any([opt.otype == dns.edns.COOKIE for + opt in res.options])) + + def testOnlyClientCookieTCP(self): + opts = [ + dns.edns.GenericOption(dns.edns.COOKIE, + b'\x22\x11\x33\x44\x55\x66\x77\x88')] + query = dns.message.make_query('www.example.org', 'A', options=opts) + res = self.sendTCPQuery(query) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertTrue(any(opt.otype == dns.edns.COOKIE for + opt in res.options)) + + + def testCorrectCookie(self): + opts = [self.getCookieFromServer()] + query = dns.message.make_query('www.example.org', 'A', options=opts) + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + + def testBrokenCookie(self): + data = self.getCookieFromServer().data + # replace a byte in the client cookie + data = data.replace(b'\x11', b'\x12') + opts = [dns.edns.GenericOption(dns.edns.COOKIE, data)] + query = dns.message.make_query('www.example.org', 'A', options=opts) + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, 23) + for opt in res.options: + if opt.otype == dns.edns.COOKIE: + self.assertNotEqual(opt.data, opts[0].data) + return + self.fail() diff --git a/regression-tests.auth-py/test_ProxyProtocol.py b/regression-tests.auth-py/test_ProxyProtocol.py new file mode 100644 index 000000000000..2d2b7ad575e3 --- /dev/null +++ b/regression-tests.auth-py/test_ProxyProtocol.py @@ -0,0 +1,239 @@ +import clientsubnetoption +import dns +import os +import socket +import struct +import threading +import time +import unittest + +from authtests import AuthTest +from proxyprotocol import ProxyProtocol + +class TestProxyProtocolLuaRecords(AuthTest): + _config_template = """ +launch=bind +any-to-tcp=no +proxy-protocol-from=127.0.0.1 +edns-subnet-processing=yes +""" + + _zones = { + 'example.org': """ +example.org. 3600 IN SOA {soa} +example.org. 3600 IN NS ns1.example.org. +example.org. 3600 IN NS ns2.example.org. +ns1.example.org. 3600 IN A {prefix}.10 +ns2.example.org. 3600 IN A {prefix}.11 + +myip.example.org. 3600 IN LUA TXT "who:toString()..'/'..bestwho:toString()" + """ + } + + @classmethod + def setUpClass(cls): + super(TestProxyProtocolLuaRecords, cls).setUpClass() + + def testWhoAmI(self): + """ + See if LUA who picks up the inner address from the PROXY protocol + """ + + for testWithECS in True, False: + # first test with an unproxied query - should get ignored + + options = [] + expectedText = '192.0.2.1/192.0.2.1' + + if testWithECS: + ecso = clientsubnetoption.ClientSubnetOption('192.0.2.5', 32) + options.append(ecso) + expectedText = '192.0.2.1/192.0.2.5' + + query = dns.message.make_query('myip.example.org', 'TXT', 'IN', use_edns=testWithECS, options=options, payload=512) + + res = self.sendUDPQuery(query) + + self.assertEqual(res, None) # query was ignored correctly + + + # now send a proxied query + queryPayload = query.to_wire() + ppPayload = ProxyProtocol.getPayload(False, False, False, "192.0.2.1", "10.1.2.3", 12345, 53, []) + payload = ppPayload + queryPayload + + # UDP + self._sock.settimeout(2.0) + + try: + self._sock.send(payload) + data = self._sock.recv(4096) + except socket.timeout: + data = None + finally: + self._sock.settimeout(None) + + res = None + if data: + res = dns.message.from_wire(data) + + expected = [dns.rrset.from_text('myip.example.org.', 0, dns.rdataclass.IN, 'TXT', expectedText)] + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertEqual(res.answer, expected) + + # TCP + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2.0) + sock.connect(("127.0.0.1", self._authPort)) + + try: + sock.send(ppPayload) + sock.send(struct.pack("!H", len(queryPayload))) + sock.send(queryPayload) + data = sock.recv(2) + if data: + (datalen,) = struct.unpack("!H", data) + data = sock.recv(datalen) + except socket.timeout as e: + print("Timeout: %s" % (str(e))) + data = None + except socket.error as e: + print("Network error: %s" % (str(e))) + data = None + finally: + sock.close() + + res = None + if data: + res = dns.message.from_wire(data) + + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertEqual(res.answer, expected) + +class TestProxyProtocolNOTIFY(AuthTest): + _config_template = """ +launch=bind +any-to-tcp=no +proxy-protocol-from=127.0.0.1 +secondary +""" + + _zones = { 'example.org': '192.0.2.1', + 'example.com': '192.0.2.2' + } + + @classmethod + def generateAuthZone(cls, confdir, zonename, zonecontent): + try: + os.unlink(os.path.join(confdir, '%s.zone' % zonename)) + except: + pass + + @classmethod + def generateAuthNamedConf(cls, confdir, zones): + with open(os.path.join(confdir, 'named.conf'), 'w') as namedconf: + namedconf.write(""" +options { + directory "%s"; +};""" % confdir) + for zonename in zones: + zone = '.' if zonename == 'ROOT' else zonename + + namedconf.write(""" + zone "%s" { + type slave; + file "%s.zone"; + masters { %s; }; + };""" % (zone, zonename, cls._zones[zone])) + + + @classmethod + def setUpClass(cls): + super(TestProxyProtocolNOTIFY, cls).setUpClass() + + def testNOTIFY(self): + """ + Check that NOTIFY is properly accepted/rejected based on the PROXY header inner address + """ + + query = dns.message.make_query('example.org', 'SOA') + query.set_opcode(dns.opcode.NOTIFY) + + queryPayload = query.to_wire() + + for task in ('192.0.2.1', dns.rcode.NOERROR), ('192.0.2.2', dns.rcode.REFUSED): + ip, expectedrcode = task + + ppPayload = ProxyProtocol.getPayload(False, False, False, ip, "10.1.2.3", 12345, 53, []) + payload = ppPayload + queryPayload + + self._sock.settimeout(2.0) + + try: + self._sock.send(payload) + data = self._sock.recv(4096) + except socket.timeout: + data = None + finally: + self._sock.settimeout(None) + + res = None + if data: + res = dns.message.from_wire(data) + + self.assertRcodeEqual(res, expectedrcode) + + +class TestProxyProtocolAXFRACL(AuthTest): + _config_template = """ +launch=bind +any-to-tcp=no +proxy-protocol-from=127.0.0.1 +allow-axfr-ips=192.0.2.53 +""" + + @classmethod + def setUpClass(cls): + super(TestProxyProtocolAXFRACL, cls).setUpClass() + + def testAXFR(self): + """ + Check that AXFR is properly accepted/rejected based on the PROXY header inner address + """ + + query = dns.message.make_query('example.org', 'AXFR') + + queryPayload = query.to_wire() + + for task in ('192.0.2.1', dns.rcode.NOTAUTH), ('127.0.0.1', dns.rcode.NOTAUTH), ('192.0.2.53', dns.rcode.NOERROR): + ip, expectedrcode = task + + ppPayload = ProxyProtocol.getPayload(False, True, False, ip, "10.1.2.3", 12345, 53, []) + + # TCP + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(2.0) + sock.connect(("127.0.0.1", self._authPort)) + + try: + sock.send(ppPayload) + sock.send(struct.pack("!H", len(queryPayload))) + sock.send(queryPayload) + data = sock.recv(2) + if data: + (datalen,) = struct.unpack("!H", data) + data = sock.recv(datalen) + except socket.timeout as e: + print("Timeout: %s" % (str(e))) + data = None + except socket.error as e: + print("Network error: %s" % (str(e))) + data = None + finally: + sock.close() + + res = None + if data: + res = dns.message.from_wire(data) + + self.assertRcodeEqual(res, expectedrcode) diff --git a/regression-tests.auth-py/test_XFRIncomplete.py b/regression-tests.auth-py/test_XFRIncomplete.py new file mode 100644 index 000000000000..5a9c10a648f8 --- /dev/null +++ b/regression-tests.auth-py/test_XFRIncomplete.py @@ -0,0 +1,197 @@ +import dns +import json +import os +import requests +import socket +import struct +import sys +import threading +import time + +from authtests import AuthTest + +class BadXFRServer(object): + + def __init__(self, port): + self._currentSerial = 0 + self._targetSerial = 1 + self._serverPort = port + listener = threading.Thread(name='XFR Listener', target=self._listener, args=[]) + listener.setDaemon(True) + listener.start() + + def getCurrentSerial(self): + return self._currentSerial + + def moveToSerial(self, newSerial): + if newSerial == self._currentSerial: + return False + + #if newSerial != self._currentSerial + 1: + # raise AssertionError("Asking the XFR server to serve serial %d, already serving %d" % (newSerial, self._currentSerial)) + self._targetSerial = newSerial + return True + + def _getAnswer(self, message): + + response = dns.message.make_response(message) + records = [] + + if message.question[0].rdtype == dns.rdatatype.AXFR: + if self._currentSerial != 0: + print('Received an AXFR query but IXFR expected because the current serial is %d' % (self._currentSerial)) + return (None, self._currentSerial) + + newSerial = self._targetSerial + records = [ + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('a.example.zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1'), + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial) + ] + + elif message.question[0].rdtype == dns.rdatatype.IXFR: + oldSerial = message.authority[0][0].serial + + newSerial = self._targetSerial + if newSerial == 2: + records = [ + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % oldSerial), + # no deletion + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('b.example.zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1'), + ] + elif newSerial == 3: + records = [ + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('a.example.zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1'), + ] + + response.answer = records + return (newSerial, response) + + def _connectionHandler(self, conn): + data = None + while True: + data = conn.recv(2) + if not data: + break + (datalen,) = struct.unpack("!H", data) + data = conn.recv(datalen) + if not data: + break + + message = dns.message.from_wire(data) + if len(message.question) != 1: + print('Invalid query, qdcount is %d' % (len(message.question)), file=sys.stderr) + break + if not message.question[0].rdtype in [dns.rdatatype.AXFR, dns.rdatatype.IXFR]: + print('Invalid query, qtype is %d' % (message.question.rdtype), file=sys.stderr) + break + (serial, answer) = self._getAnswer(message) + if not answer: + print('Unable to get a response for %s %d' % (message.question[0].name, message.question[0].rdtype), file=sys.stderr) + break + + wire = answer.to_wire() + conn.send(struct.pack("!H", len(wire))) + conn.send(wire) + self._currentSerial = serial + break + + conn.close() + + def _listener(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + try: + sock.bind(("127.0.0.1", self._serverPort)) + except socket.error as e: + print("Error binding in the IXFR listener: %s" % str(e)) + sys.exit(1) + + sock.listen(100) + while True: + try: + (conn, _) = sock.accept() + thread = threading.Thread(name='IXFR Connection Handler', + target=self._connectionHandler, + args=[conn]) + thread.setDaemon(True) + thread.start() + + except socket.error as e: + print('Error in IXFR socket: %s' % str(e)) + sock.close() + +badxfrServerPort = 4251 +badxfrServer = BadXFRServer(badxfrServerPort) + +class XFRIncompleteAuthTest(AuthTest): + """ + This test makes sure that we correctly detect incomplete RPZ zones via AXFR then IXFR + """ + + global badxfrServerPort + _config_template = """ +launch=gsqlite3 bind +gsqlite3-database=configs/auth/powerdns.sqlite +gsqlite3-dnssec +slave +cache-ttl=0 +query-cache-ttl=0 +domain-metadata-cache-ttl=0 +negquery-cache-ttl=0 +slave-cycle-interval=1 +""" + + @classmethod + def setUpClass(cls): + super(XFRIncompleteAuthTest, cls).setUpClass() + os.system("$PDNSUTIL --config-dir=configs/auth create-slave-zone zone.rpz. 127.0.0.1:%s" % (badxfrServerPort,)) + os.system("$PDNSUTIL --config-dir=configs/auth set-meta zone.rpz. IXFR 1") + + def waitUntilCorrectSerialIsLoaded(self, serial, timeout=5): + global badxfrServer + + badxfrServer.moveToSerial(serial) + + attempts = 0 + while attempts < timeout: + currentSerial = badxfrServer.getCurrentSerial() + if currentSerial > serial: + raise AssertionError("Expected serial %d, got %d" % (serial, currentSerial)) + if currentSerial == serial: + return + + attempts = attempts + 1 + time.sleep(1) + + raise AssertionError("Waited %d seconds for the serial to be updated to %d but the serial is still %d" % (timeout, serial, currentSerial)) + + def checkZone(self): + query = dns.message.make_query('zone.rpz.', 'SOA') + res = self.sendUDPQuery(query) # , count=len(expected)) + + expected = [dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. 1 3600 3600 3600 1')] + self.assertEqual(res.answer, expected) + + def doRetrieve(self): + os.system("$PDNSCONTROL --socket-dir=configs/auth retrieve zone.rpz.") + + def testXFR(self): + # self.waitForTCPSocket("127.0.0.1", self._wsPort) + # First zone + self.doRetrieve() + self.waitUntilCorrectSerialIsLoaded(1) + self.checkZone() + + # second zone, should fail, incomplete IXFR + self.doRetrieve() + self.waitUntilCorrectSerialIsLoaded(2) + self.checkZone() + + # third zone, should fail, incomplete AXFR + self.doRetrieve() + self.waitUntilCorrectSerialIsLoaded(3) + self.checkZone() diff --git a/regression-tests.dnsdist/cookiesoption.py b/regression-tests.dnsdist/cookiesoption.py index d977e1030db6..8e4016d27b07 100644 --- a/regression-tests.dnsdist/cookiesoption.py +++ b/regression-tests.dnsdist/cookiesoption.py @@ -96,3 +96,5 @@ def __ne__(self, other): dns.edns._type_to_class[0x000A] = CookiesOption + +dns.rcode.BADCOOKIE = 23 diff --git a/regression-tests.dnsdist/dnsdisttests.py b/regression-tests.dnsdist/dnsdisttests.py index 7284122ec7ef..011bae3bbf36 100644 --- a/regression-tests.dnsdist/dnsdisttests.py +++ b/regression-tests.dnsdist/dnsdisttests.py @@ -10,13 +10,21 @@ import threading import time import unittest + import clientsubnetoption + import dns import dns.message + import libnacl import libnacl.utils +import h2.connection +import h2.events +import h2.config + from eqdnsmessage import AssertEqualDNSMessageMixin +from proxyprotocol import ProxyProtocol # Python2/3 compatibility hacks try: @@ -81,6 +89,7 @@ def startDNSDist(cls): with open(confFile, 'w') as conf: conf.write("-- Autogenerated by dnsdisttests.py\n") conf.write(cls._config_template % params) + conf.write("setSecurityPollSuffix('')") if cls._skipListeningOnCL: dnsdistcmd = [os.environ['DNSDISTBIN'], '--supervised', '-C', confFile ] @@ -231,12 +240,68 @@ def UDPResponder(cls, port, fromQueue, toQueue, trailingDataResponse=False, call sock.close() @classmethod - def TCPResponder(cls, port, fromQueue, toQueue, trailingDataResponse=False, multipleResponses=False, callback=None, tlsContext=None): + def handleTCPConnection(cls, conn, fromQueue, toQueue, trailingDataResponse=False, multipleResponses=False, callback=None): + ignoreTrailing = trailingDataResponse is True + data = conn.recv(2) + if not data: + conn.close() + return + + (datalen,) = struct.unpack("!H", data) + data = conn.recv(datalen) + forceRcode = None + try: + request = dns.message.from_wire(data, ignore_trailing=ignoreTrailing) + except dns.message.TrailingJunk as e: + if trailingDataResponse is False or forceRcode is True: + raise + print("TCP query with trailing data, synthesizing response") + request = dns.message.from_wire(data, ignore_trailing=True) + forceRcode = trailingDataResponse + + if callback: + wire = callback(request) + else: + response = cls._getResponse(request, fromQueue, toQueue, synthesize=forceRcode) + if response: + wire = response.to_wire(max_size=65535) + + if not wire: + conn.close() + return + + conn.send(struct.pack("!H", len(wire))) + conn.send(wire) + + while multipleResponses: + # do not block, and stop as soon as the queue is empty, either the next response is already here or we are done + # otherwise we might read responses intended for the next connection + if fromQueue.empty(): + break + + response = fromQueue.get(False) + if not response: + break + + response = copy.copy(response) + response.id = request.id + wire = response.to_wire(max_size=65535) + try: + conn.send(struct.pack("!H", len(wire))) + conn.send(wire) + except socket.error as e: + # some of the tests are going to close + # the connection on us, just deal with it + break + + conn.close() + + @classmethod + def TCPResponder(cls, port, fromQueue, toQueue, trailingDataResponse=False, multipleResponses=False, callback=None, tlsContext=None, multipleConnections=False): # trailingDataResponse=True means "ignore trailing data". # Other values are either False (meaning "raise an exception") # or are interpreted as a response RCODE for queries with trailing data. # callback is invoked for every -even healthcheck ones- query and should return a raw response - ignoreTrailing = trailingDataResponse is True sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -256,58 +321,147 @@ def TCPResponder(cls, port, fromQueue, toQueue, trailingDataResponse=False, mult (conn, _) = sock.accept() except ssl.SSLError: continue + except ConnectionResetError: + continue + conn.settimeout(5.0) - data = conn.recv(2) - if not data: - conn.close() - continue + if multipleConnections: + thread = threading.Thread(name='TCP Connection Handler', + target=cls.handleTCPConnection, + args=[conn, fromQueue, toQueue, trailingDataResponse, multipleResponses, callback]) + thread.setDaemon(True) + thread.start() + else: + cls.handleTCPConnection(conn, fromQueue, toQueue, trailingDataResponse, multipleResponses, callback) - (datalen,) = struct.unpack("!H", data) - data = conn.recv(datalen) - forceRcode = None - try: - request = dns.message.from_wire(data, ignore_trailing=ignoreTrailing) - except dns.message.TrailingJunk as e: - if trailingDataResponse is False or forceRcode is True: - raise - print("TCP query with trailing data, synthesizing response") - request = dns.message.from_wire(data, ignore_trailing=True) - forceRcode = trailingDataResponse + sock.close() - if callback: - wire = callback(request) - else: - response = cls._getResponse(request, fromQueue, toQueue, synthesize=forceRcode) - if response: - wire = response.to_wire(max_size=65535) + @classmethod + def handleDoHConnection(cls, config, conn, fromQueue, toQueue, trailingDataResponse, multipleResponses, callback, tlsContext, useProxyProtocol): + ignoreTrailing = trailingDataResponse is True + h2conn = h2.connection.H2Connection(config=config) + h2conn.initiate_connection() + conn.sendall(h2conn.data_to_send()) + dnsData = {} + + if useProxyProtocol: + # try to read the entire Proxy Protocol header + proxy = ProxyProtocol() + header = conn.recv(proxy.HEADER_SIZE) + if not header: + print('unable to get header') + conn.close() + return - if not wire: + if not proxy.parseHeader(header): + print('unable to parse header') + print(header) conn.close() - continue + return - conn.send(struct.pack("!H", len(wire))) - conn.send(wire) + proxyContent = conn.recv(proxy.contentLen) + if not proxyContent: + print('unable to get content') + conn.close() + return - while multipleResponses: - if fromQueue.empty(): - break + payload = header + proxyContent + toQueue.put(payload, True, cls._queueTimeout) - response = fromQueue.get(True, cls._queueTimeout) - if not response: - break + # be careful, HTTP/2 headers and data might be in different recv() results + requestHeaders = None + while True: + data = conn.recv(65535) + if not data: + break + + events = h2conn.receive_data(data) + for event in events: + if isinstance(event, h2.events.RequestReceived): + requestHeaders = event.headers + if isinstance(event, h2.events.DataReceived): + h2conn.acknowledge_received_data(event.flow_controlled_length, event.stream_id) + if not event.stream_id in dnsData: + dnsData[event.stream_id] = b'' + dnsData[event.stream_id] = dnsData[event.stream_id] + (event.data) + if event.stream_ended: + forceRcode = None + status = 200 + try: + request = dns.message.from_wire(dnsData[event.stream_id], ignore_trailing=ignoreTrailing) + except dns.message.TrailingJunk as e: + if trailingDataResponse is False or forceRcode is True: + raise + print("DOH query with trailing data, synthesizing response") + request = dns.message.from_wire(dnsData[event.stream_id], ignore_trailing=True) + forceRcode = trailingDataResponse + + if callback: + status, wire = callback(request, requestHeaders, fromQueue, toQueue) + else: + response = cls._getResponse(request, fromQueue, toQueue, synthesize=forceRcode) + if response: + wire = response.to_wire(max_size=65535) + + if not wire: + conn.close() + conn = None + break + + headers = [ + (':status', str(status)), + ('content-length', str(len(wire))), + ('content-type', 'application/dns-message'), + ] + h2conn.send_headers(stream_id=event.stream_id, headers=headers) + h2conn.send_data(stream_id=event.stream_id, data=wire, end_stream=True) + + data_to_send = h2conn.data_to_send() + if data_to_send: + conn.sendall(data_to_send) + + if conn is None: + break + + if conn is not None: + conn.close() - response = copy.copy(response) - response.id = request.id - wire = response.to_wire(max_size=65535) - try: - conn.send(struct.pack("!H", len(wire))) - conn.send(wire) - except socket.error as e: - # some of the tests are going to close - # the connection on us, just deal with it - break + @classmethod + def DOHResponder(cls, port, fromQueue, toQueue, trailingDataResponse=False, multipleResponses=False, callback=None, tlsContext=None, useProxyProtocol=False): + # trailingDataResponse=True means "ignore trailing data". + # Other values are either False (meaning "raise an exception") + # or are interpreted as a response RCODE for queries with trailing data. + # callback is invoked for every -even healthcheck ones- query and should return a raw response - conn.close() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + try: + sock.bind(("127.0.0.1", port)) + except socket.error as e: + print("Error binding in the TCP responder: %s" % str(e)) + sys.exit(1) + + sock.listen(100) + if tlsContext: + sock = tlsContext.wrap_socket(sock, server_side=True) + + config = h2.config.H2Configuration(client_side=False) + + while True: + try: + (conn, _) = sock.accept() + except ssl.SSLError: + continue + except ConnectionResetError: + continue + + conn.settimeout(5.0) + thread = threading.Thread(name='DoH Connection Handler', + target=cls.handleDoHConnection, + args=[config, conn, fromQueue, toQueue, trailingDataResponse, multipleResponses, callback, tlsContext, useProxyProtocol]) + thread.setDaemon(True) + thread.start() sock.close() @@ -383,18 +537,25 @@ def sendTCPQueryOverConnection(cls, sock, query, rawQuery=False, response=None, @classmethod def recvTCPResponseOverConnection(cls, sock, useQueue=False, timeout=2.0): + print("reading data") message = None data = sock.recv(2) if data: (datalen,) = struct.unpack("!H", data) + print(datalen) data = sock.recv(datalen) if data: + print(data) message = dns.message.from_wire(data) + print(useQueue) if useQueue and not cls._fromResponderQueue.empty(): receivedQuery = cls._fromResponderQueue.get(True, timeout) + print("Got from queue") + print(receivedQuery) return (receivedQuery, message) else: + print("queue empty") return message @classmethod @@ -409,15 +570,20 @@ def sendTCPQuery(cls, query, response, useQueue=True, timeout=2.0, rawQuery=Fals cls.sendTCPQueryOverConnection(sock, query, rawQuery) message = cls.recvTCPResponseOverConnection(sock) except socket.timeout as e: - print("Timeout: %s" % (str(e))) + print("Timeout while sending or receiving TCP data: %s" % (str(e))) except socket.error as e: print("Network error: %s" % (str(e))) finally: sock.close() receivedQuery = None + print(useQueue) if useQueue and not cls._fromResponderQueue.empty(): + print("Got from queue") + print(receivedQuery) receivedQuery = cls._fromResponderQueue.get(True, timeout) + else: + print("queue is empty") return (receivedQuery, message) @@ -451,7 +617,7 @@ def sendTCPQueryWithMultipleResponses(cls, query, responses, useQueue=True, time messages.append(dns.message.from_wire(data)) except socket.timeout as e: - print("Timeout: %s" % (str(e))) + print("Timeout while receiving multiple TCP responses: %s" % (str(e))) except socket.error as e: print("Network error: %s" % (str(e))) finally: @@ -463,7 +629,7 @@ def sendTCPQueryWithMultipleResponses(cls, query, responses, useQueue=True, time return (receivedQuery, messages) def setUp(self): - # This function is called before every tests + # This function is called before every test # Clear the responses counters self._responsesCounter.clear() @@ -472,11 +638,7 @@ def setUp(self): # Make sure the queues are empty, in case # a previous test failed - while not self._toResponderQueue.empty(): - self._toResponderQueue.get(False) - - while not self._fromResponderQueue.empty(): - self._fromResponderQueue.get(False) + self.clearResponderQueues() super(DNSDistTest, self).setUp() @@ -589,9 +751,19 @@ def checkMessageEDNSWithECS(self, expected, received, additionalOptions=0): self.compareOptions(expected.options, received.options) self.assertTrue(hasECS) + def checkMessageEDNS(self, expected, received): + self.assertEqual(expected, received) + self.assertEqual(received.edns, 0) + self.assertEqual(expected.payload, received.payload) + self.assertEqual(len(expected.options), len(received.options)) + self.compareOptions(expected.options, received.options) + def checkQueryEDNSWithECS(self, expected, received, additionalOptions=0): self.checkMessageEDNSWithECS(expected, received, additionalOptions) + def checkQueryEDNS(self, expected, received): + self.checkMessageEDNS(expected, received) + def checkResponseEDNSWithECS(self, expected, received, additionalOptions=0): self.checkMessageEDNSWithECS(expected, received, additionalOptions) @@ -628,3 +800,33 @@ def generateNewCertificateAndKey(self): for inFileName in ['server.pem', 'ca.pem']: with open(inFileName) as inFile: outFile.write(inFile.read()) + + def checkMessageProxyProtocol(self, receivedProxyPayload, source, destination, isTCP, values=[], v6=False, sourcePort=None, destinationPort=None): + proxy = ProxyProtocol() + self.assertTrue(proxy.parseHeader(receivedProxyPayload)) + self.assertEqual(proxy.version, 0x02) + self.assertEqual(proxy.command, 0x01) + if v6: + self.assertEqual(proxy.family, 0x02) + else: + self.assertEqual(proxy.family, 0x01) + if not isTCP: + self.assertEqual(proxy.protocol, 0x02) + else: + self.assertEqual(proxy.protocol, 0x01) + self.assertGreater(proxy.contentLen, 0) + + self.assertTrue(proxy.parseAddressesAndPorts(receivedProxyPayload)) + self.assertEqual(proxy.source, source) + self.assertEqual(proxy.destination, destination) + if sourcePort: + self.assertEqual(proxy.sourcePort, sourcePort) + if destinationPort: + self.assertEqual(proxy.destinationPort, destinationPort) + else: + self.assertEqual(proxy.destinationPort, self._dnsDistPort) + + self.assertTrue(proxy.parseAdditionalValues(receivedProxyPayload)) + proxy.values.sort() + values.sort() + self.assertEqual(proxy.values, values) diff --git a/regression-tests.dnsdist/requirements.txt b/regression-tests.dnsdist/requirements.txt index afa9fa76ecd5..fd27b154cb54 100644 --- a/regression-tests.dnsdist/requirements.txt +++ b/regression-tests.dnsdist/requirements.txt @@ -8,3 +8,4 @@ future>=0.17.1 pycurl>=7.43.0 lmdb>=0.95 cdbx==0.1.2 +h2>=4.0.0 diff --git a/regression-tests.dnsdist/runtests b/regression-tests.dnsdist/runtests index dcdc5441e57e..a1d63945e513 100755 --- a/regression-tests.dnsdist/runtests +++ b/regression-tests.dnsdist/runtests @@ -49,12 +49,19 @@ openssl x509 -req -days 1 -CA ca.pem -CAkey ca.key -CAcreateserial -in server.cs # Generate a chain cat server.pem ca.pem > server.chain -if ! nosetests --with-xunit $@; then +out=$(mktemp) +set -o pipefail +if ! nosetests --with-xunit $@ 2>&1 | tee "${out}" ; then for log in configs/*.log; do echo "=== ${log} ===" cat "${log}" + echo done + echo "=== nosetests log ===" + cat "${out}" + echo "=== end of nosetests log ===" false fi +rm -f "${out}" rm -f ca.key ca.pem ca.srl server.csr server.key server.pem server.chain server.ocsp diff --git a/regression-tests.dnsdist/test_API.py b/regression-tests.dnsdist/test_API.py index 76d13f26e93f..98141fdc0de6 100644 --- a/regression-tests.dnsdist/test_API.py +++ b/regression-tests.dnsdist/test_API.py @@ -8,19 +8,15 @@ import time from dnsdisttests import DNSDistTest -class TestAPIBasics(DNSDistTest): - +class APITestsBase(DNSDistTest): + __test__ = False _webTimeout = 2.0 _webServerPort = 8083 _webServerBasicAuthPassword = 'secret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' _webServerAPIKey = 'apisecret' - # paths accessible using the API key only - _apiOnlyPaths = ['/api/v1/servers/localhost/config', '/api/v1/servers/localhost/config/allow-from', '/api/v1/servers/localhost/statistics'] - # paths accessible using an API key or basic auth - _statsPaths = [ '/jsonstat?command=stats', '/jsonstat?command=dynblocklist', '/api/v1/servers/localhost'] - # paths accessible using basic auth only (list not exhaustive) - _basicOnlyPaths = ['/', '/index.html'] - _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' + _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setACL({"127.0.0.1/32", "::1/128"}) newServer{address="127.0.0.1:%s", pool={'', 'mypool'}} @@ -28,6 +24,16 @@ class TestAPIBasics(DNSDistTest): setWebserverConfig({password="%s", apiKey="%s"}) """ +class TestAPIBasics(APITestsBase): + + # paths accessible using the API key only + _apiOnlyPaths = ['/api/v1/servers/localhost/config', '/api/v1/servers/localhost/config/allow-from', '/api/v1/servers/localhost/statistics'] + # paths accessible using an API key or basic auth + _statsPaths = [ '/jsonstat?command=stats', '/jsonstat?command=dynblocklist', '/api/v1/servers/localhost'] + # paths accessible using basic auth only (list not exhaustive) + _basicOnlyPaths = ['/', '/index.html'] + __test__ = True + def testBasicAuth(self): """ API: Basic Authentication @@ -278,8 +284,11 @@ def testServersLocalhostStatistics(self): 'noncompliant-responses', 'rdqueries', 'empty-queries', 'cache-hits', 'cache-misses', 'cpu-iowait', 'cpu-steal', 'cpu-sys-msec', 'cpu-user-msec', 'fd-usage', 'dyn-blocked', 'dyn-block-nmg-size', 'rule-servfail', 'rule-truncated', 'security-status', - 'udp-in-errors', 'udp-noport-errors', 'udp-recvbuf-errors', 'udp-sndbuf-errors', - 'doh-query-pipe-full', 'doh-response-pipe-full', 'proxy-protocol-invalid', 'tcp-listen-overflows'] + 'udp-in-csum-errors', 'udp-in-errors', 'udp-noport-errors', 'udp-recvbuf-errors', 'udp-sndbuf-errors', + 'udp6-in-errors', 'udp6-recvbuf-errors', 'udp6-sndbuf-errors', 'udp6-noport-errors', 'udp6-in-csum-errors', + 'doh-query-pipe-full', 'doh-response-pipe-full', 'proxy-protocol-invalid', 'tcp-listen-overflows', + 'outgoing-doh-query-pipe-full', 'tcp-query-pipe-full', 'tcp-cross-protocol-query-pipe-full', + 'tcp-cross-protocol-response-pipe-full'] for key in expected: self.assertIn(key, values) @@ -335,14 +344,8 @@ def testJsonstatDynblocklist(self): for key in ['blocks']: self.assertTrue(content[key] >= 0) -class TestAPIServerDown(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' - _webServerAPIKey = 'apisecret' - # paths accessible using the API key - _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] +class TestAPIServerDown(APITestsBase): + __test__ = True _config_template = """ setACL({"127.0.0.1/32", "::1/128"}) newServer{address="127.0.0.1:%s"} @@ -365,14 +368,10 @@ def testServerDownNoLatencyLocalhost(self): self.assertEqual(content['servers'][0]['latency'], None) -class TestAPIWritable(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' - _webServerAPIKey = 'apisecret' +class TestAPIWritable(APITestsBase): + __test__ = True _APIWriteDir = '/tmp' - _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey', '_APIWriteDir'] + _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed', '_APIWriteDir'] _config_template = """ setACL({"127.0.0.1/32", "::1/128"}) newServer{address="127.0.0.1:%s"} @@ -438,19 +437,15 @@ def testSetACL(self): """setACL({"203.0.113.0/24", "198.51.100.0/24", "192.0.2.0/24"})\n""" }) -class TestAPICustomHeaders(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' - _webServerAPIKey = 'apisecret' +class TestAPICustomHeaders(APITestsBase): + __test__ = True # paths accessible using the API key only _apiOnlyPath = '/api/v1/servers/localhost/config' # paths accessible using basic auth only (list not exhaustive) _basicOnlyPath = '/' _consoleKey = DNSDistTest.generateConsoleKey() _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') - _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setKey("%s") controlSocket("127.0.0.1:%s") @@ -486,12 +481,8 @@ def testBasicHeadersUpdate(self): self.assertEqual(r.headers.get('x-powered-by'), "dnsdist") self.assertTrue("x-frame-options" in r.headers) -class TestStatsWithoutAuthentication(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' - _webServerAPIKey = 'apisecret' +class TestStatsWithoutAuthentication(APITestsBase): + __test__ = True # paths accessible using the API key only _apiOnlyPath = '/api/v1/servers/localhost/config' # paths accessible using basic auth only (list not exhaustive) @@ -499,7 +490,7 @@ class TestStatsWithoutAuthentication(DNSDistTest): _noAuthenticationPaths = [ '/metrics', '/jsonstat?command=dynblocklist' ] _consoleKey = DNSDistTest.generateConsoleKey() _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') - _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setKey("%s") controlSocket("127.0.0.1:%s") @@ -544,21 +535,19 @@ def testAuth(self): self.assertTrue(r) self.assertEqual(r.status_code, 200) -class TestAPIAuth(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' +class TestAPIAuth(APITestsBase): + __test__ = True _webServerBasicAuthPasswordNew = 'password' - _webServerAPIKey = 'apisecret' + _webServerBasicAuthPasswordNewHashed = '$scrypt$ln=10,p=1,r=8$yefz8SAuT3lj3moXqUYvmw==$T98/RYMp76ZYNjd7MpAkcVXZEDqpLtrc3tQ52QflVBA=' _webServerAPIKeyNew = 'apipassword' + _webServerAPIKeyNewHashed = '$scrypt$ln=9,p=1,r=8$y96I9nfkY0LWDQEdSUzWgA==$jiyn9QD36o9d0ADrlqiIBk4AKyQrkD1KYw3CexwtHp4=' # paths accessible using the API key only _apiOnlyPath = '/api/v1/servers/localhost/config' # paths accessible using basic auth only (list not exhaustive) _basicOnlyPath = '/' _consoleKey = DNSDistTest.generateConsoleKey() _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') - _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setKey("%s") controlSocket("127.0.0.1:%s") @@ -574,7 +563,7 @@ def testBasicAuthChange(self): """ url = 'http://127.0.0.1:' + str(self._webServerPort) + self._basicOnlyPath - self.sendConsoleCommand('setWebserverConfig({{password="{}"}})'.format(self._webServerBasicAuthPasswordNew)) + self.sendConsoleCommand('setWebserverConfig({{password="{}"}})'.format(self._webServerBasicAuthPasswordNewHashed)) r = requests.get(url, auth=('whatever', self._webServerBasicAuthPasswordNew), timeout=self._webTimeout) self.assertTrue(r) @@ -590,7 +579,7 @@ def testXAPIKeyChange(self): """ url = 'http://127.0.0.1:' + str(self._webServerPort) + self._apiOnlyPath - self.sendConsoleCommand('setWebserverConfig({{apiKey="{}"}})'.format(self._webServerAPIKeyNew)) + self.sendConsoleCommand('setWebserverConfig({{apiKey="{}"}})'.format(self._webServerAPIKeyNewHashed)) headers = {'x-api-key': self._webServerAPIKeyNew} r = requests.get(url, headers=headers, timeout=self._webTimeout) @@ -608,7 +597,7 @@ def testBasicAuthOnlyChange(self): """ url = 'http://127.0.0.1:' + str(self._webServerPort) + self._apiOnlyPath - self.sendConsoleCommand('setWebserverConfig({{apiKey="{}"}})'.format(self._webServerAPIKeyNew)) + self.sendConsoleCommand('setWebserverConfig({{apiKey="{}"}})'.format(self._webServerAPIKeyNewHashed)) headers = {'x-api-key': self._webServerAPIKeyNew} r = requests.get(url, headers=headers, timeout=self._webTimeout) @@ -621,15 +610,11 @@ def testBasicAuthOnlyChange(self): r = requests.get(url, headers=headers, timeout=self._webTimeout) self.assertEqual(r.status_code, 401) -class TestAPIACL(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' - _webServerAPIKey = 'apisecret' +class TestAPIACL(APITestsBase): + __test__ = True _consoleKey = DNSDistTest.generateConsoleKey() _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') - _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setKey("%s") controlSocket("127.0.0.1:%s") @@ -658,15 +643,13 @@ def testACLChange(self): self.assertTrue(r) self.assertEqual(r.status_code, 200) -class TestCustomLuaEndpoint(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' +class TestCustomLuaEndpoint(APITestsBase): + __test__ = True _config_template = """ setACL({"127.0.0.1/32", "::1/128"}) newServer{address="127.0.0.1:%s"} - webserver("127.0.0.1:%s", "%s") + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s"}) function customHTTPHandler(req, resp) if req.path ~= '/foo' then @@ -702,7 +685,7 @@ class TestCustomLuaEndpoint(DNSDistTest): end registerWebHandler('/foo', customHTTPHandler) """ - _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPassword'] + _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed'] def testBasic(self): """ @@ -716,15 +699,11 @@ def testBasic(self): self.assertEqual(r.content, b'It works!') self.assertEqual(r.headers.get('foo'), "Bar") -class TestWebConcurrentConnectionsL(DNSDistTest): - - _webTimeout = 2.0 - _webServerPort = 8083 - _webServerBasicAuthPassword = 'secret' - _webServerAPIKey = 'apisecret' +class TestWebConcurrentConnections(APITestsBase): + __test__ = True _maxConns = 2 - _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey', '_maxConns'] + _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed', '_maxConns'] _config_template = """ newServer{address="127.0.0.1:%s"} webserver("127.0.0.1:%s") diff --git a/regression-tests.dnsdist/test_AXFR.py b/regression-tests.dnsdist/test_AXFR.py index c4e43c7bad32..b1f3910788c2 100644 --- a/regression-tests.dnsdist/test_AXFR.py +++ b/regression-tests.dnsdist/test_AXFR.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import threading +import time import dns from dnsdisttests import DNSDistTest @@ -13,6 +14,8 @@ class TestAXFR(DNSDistTest): _config_template = """ newServer{address="127.0.0.1:%s"} """ + _verboseMode = True + @classmethod def startResponders(cls): print("Launching responders..") @@ -20,10 +23,17 @@ def startResponders(cls): cls._UDPResponder = threading.Thread(name='UDP Responder', target=cls.UDPResponder, args=[cls._testServerPort, cls._toResponderQueue, cls._fromResponderQueue]) cls._UDPResponder.setDaemon(True) cls._UDPResponder.start() - cls._TCPResponder = threading.Thread(name='TCP Responder', target=cls.TCPResponder, args=[cls._testServerPort, cls._toResponderQueue, cls._fromResponderQueue, False, True]) + cls._TCPResponder = threading.Thread(name='TCP Responder', target=cls.TCPResponder, args=[cls._testServerPort, cls._toResponderQueue, cls._fromResponderQueue, False, True, None, None, True]) cls._TCPResponder.setDaemon(True) cls._TCPResponder.start() + def setUp(self): + # This function is called before every test + super(TestAXFR, self).setUp() + # make sure the TCP connection handlers from the previous test + # are not going to read our queue + time.sleep(1) + def testOneMessageAXFR(self): """ AXFR: One message @@ -244,7 +254,7 @@ def testThreePlusTrailingIXFR(self): dns.rdatatype.SOA, 'ns.' + name + ' hostmaster.' + name + ' 3 3600 3600 3600 60') - # the final SOA starts the AXFR, with first an update from 1 to 2 (one removal, two additions) + # the final SOA starts the IXFR, with first an update from 1 to 2 (one removal, two additions) response = dns.message.make_response(query) response.answer.append(finalSoa) # update from 1 to 2 diff --git a/regression-tests.dnsdist/test_Advanced.py b/regression-tests.dnsdist/test_Advanced.py index 27a9f6b0f534..db0cbf7d4c90 100644 --- a/regression-tests.dnsdist/test_Advanced.py +++ b/regression-tests.dnsdist/test_Advanced.py @@ -4,8 +4,10 @@ import os import string import time +import unittest import dns import clientsubnetoption +import cookiesoption from dnsdisttests import DNSDistTest class TestAdvancedAllow(DNSDistTest): @@ -162,7 +164,6 @@ class TestAdvancedAddCD(DNSDistTest): _config_template = """ addAction("setcd.advanced.tests.powerdns.com.", SetDisableValidationAction()) - addAction(makeRule("setcdviaaction.advanced.tests.powerdns.com."), SetDisableValidationAction()) newServer{address="127.0.0.1:%s"} """ @@ -195,35 +196,6 @@ def testAdvancedSetCD(self): self.assertEqual(expectedQuery, receivedQuery) self.assertEqual(response, receivedResponse) - def testAdvancedSetCDViaAction(self): - """ - Advanced: Set CD via Action - - Send a query with CD cleared, - check that dnsdist set the CD flag. - """ - name = 'setcdviaaction.advanced.tests.powerdns.com.' - query = dns.message.make_query(name, 'A', 'IN') - expectedQuery = dns.message.make_query(name, 'A', 'IN') - expectedQuery.flags |= dns.flags.CD - - response = dns.message.make_response(query) - rrset = dns.rrset.from_text(name, - 3600, - dns.rdataclass.IN, - dns.rdatatype.A, - '127.0.0.1') - response.answer.append(rrset) - - for method in ("sendUDPQuery", "sendTCPQuery"): - sender = getattr(self, method) - (receivedQuery, receivedResponse) = sender(query, response) - self.assertTrue(receivedQuery) - self.assertTrue(receivedResponse) - receivedQuery.id = expectedQuery.id - self.assertEqual(expectedQuery, receivedQuery) - self.assertEqual(response, receivedResponse) - def testAdvancedKeepNoCD(self): """ Advanced: Preserve CD canary @@ -255,7 +227,6 @@ class TestAdvancedClearRD(DNSDistTest): _config_template = """ addAction("clearrd.advanced.tests.powerdns.com.", SetNoRecurseAction()) - addAction(makeRule("clearrdviaaction.advanced.tests.powerdns.com."), SetNoRecurseAction()) newServer{address="127.0.0.1:%s"} """ @@ -288,35 +259,6 @@ def testAdvancedClearRD(self): self.assertEqual(expectedQuery, receivedQuery) self.assertEqual(response, receivedResponse) - def testAdvancedClearRDViaAction(self): - """ - Advanced: Clear RD via Action - - Send a query with RD set, - check that dnsdist clears the RD flag. - """ - name = 'clearrdviaaction.advanced.tests.powerdns.com.' - query = dns.message.make_query(name, 'A', 'IN') - expectedQuery = dns.message.make_query(name, 'A', 'IN') - expectedQuery.flags &= ~dns.flags.RD - - response = dns.message.make_response(query) - rrset = dns.rrset.from_text(name, - 3600, - dns.rdataclass.IN, - dns.rdatatype.A, - '127.0.0.1') - response.answer.append(rrset) - - for method in ("sendUDPQuery", "sendTCPQuery"): - sender = getattr(self, method) - (receivedQuery, receivedResponse) = sender(query, response) - self.assertTrue(receivedQuery) - self.assertTrue(receivedResponse) - receivedQuery.id = expectedQuery.id - self.assertEqual(expectedQuery, receivedQuery) - self.assertEqual(response, receivedResponse) - def testAdvancedKeepRD(self): """ Advanced: Preserve RD canary @@ -409,49 +351,6 @@ def testDelayed(self): self.assertEqual(response, receivedResponse) self.assertTrue((end - begin) < timedelta(0, 1)) - -class TestAdvancedTruncateAnyAndTCP(DNSDistTest): - - _config_template = """ - truncateTC(false) - addAction(AndRule({QTypeRule("ANY"), TCPRule(true)}), TCAction()) - newServer{address="127.0.0.1:%s"} - """ - def testTruncateAnyOverTCP(self): - """ - Advanced: Truncate ANY over TCP - - Send an ANY query to "anytruncatetcp.advanced.tests.powerdns.com.", - should be truncated over TCP, not over UDP (yes, it makes no sense, - deal with it). - """ - name = 'anytruncatetcp.advanced.tests.powerdns.com.' - query = dns.message.make_query(name, 'ANY', 'IN') - # dnsdist sets RA = RD for TC responses - query.flags &= ~dns.flags.RD - - response = dns.message.make_response(query) - rrset = dns.rrset.from_text(name, - 3600, - dns.rdataclass.IN, - dns.rdatatype.A, - '127.0.0.1') - - response.answer.append(rrset) - - (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response) - self.assertTrue(receivedQuery) - self.assertTrue(receivedResponse) - receivedQuery.id = query.id - self.assertEqual(query, receivedQuery) - self.assertEqual(receivedResponse, response) - - expectedResponse = dns.message.make_response(query) - expectedResponse.flags |= dns.flags.TC - - (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False) - self.assertEqual(receivedResponse, expectedResponse) - class TestAdvancedAndNot(DNSDistTest): _config_template = """ @@ -1078,6 +977,7 @@ def testAdvancedWireLengthRule(self): (_, receivedResponse) = sender(query, response=None, useQueue=False) self.assertEqual(receivedResponse, expectedResponse) +@unittest.skipIf('SKIP_INCLUDEDIR_TESTS' in os.environ, 'IncludeDir tests are disabled') class TestAdvancedIncludeDir(DNSDistTest): _config_template = """ @@ -2344,3 +2244,67 @@ def testProtocolTCP(self): receivedQuery.id = query.id self.assertEqual(receivedQuery, query) self.assertEqual(receivedResponse, response) + +class TestAdvancedSetEDNSOptionAction(DNSDistTest): + + _config_template = """ + addAction(AllRule(), SetEDNSOptionAction(10, "deadbeefdeadc0de")) + newServer{address="127.0.0.1:%s"} + """ + + def testAdvancedSetEDNSOption(self): + """ + Advanced: Set EDNS Option + """ + name = 'setednsoption.advanced.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + eco = cookiesoption.CookiesOption(b'deadbeef', b'deadc0de') + expectedQuery = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=512, options=[eco]) + + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 3600, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + response.answer.append(rrset) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (receivedQuery, receivedResponse) = sender(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = expectedQuery.id + self.assertEqual(expectedQuery, receivedQuery) + self.checkResponseNoEDNS(response, receivedResponse) + self.checkQueryEDNS(expectedQuery, receivedQuery) + + def testAdvancedSetEDNSOptionOverwrite(self): + """ + Advanced: Set EDNS Option overwrites an existing option + """ + name = 'setednsoption-overwrite.advanced.tests.powerdns.com.' + initialECO = cookiesoption.CookiesOption(b'aaaaaaaa', b'bbbbbbbb') + query = dns.message.make_query(name, 'A', 'IN') + + overWrittenECO = cookiesoption.CookiesOption(b'deadbeef', b'deadc0de') + expectedQuery = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=512, options=[overWrittenECO]) + + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 3600, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + response.answer.append(rrset) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (receivedQuery, receivedResponse) = sender(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = expectedQuery.id + self.assertEqual(expectedQuery, receivedQuery) + self.checkResponseNoEDNS(response, receivedResponse) + self.checkQueryEDNS(expectedQuery, receivedQuery) diff --git a/regression-tests.dnsdist/test_Basics.py b/regression-tests.dnsdist/test_Basics.py index c879566dc925..7ba61ef27b35 100644 --- a/regression-tests.dnsdist/test_Basics.py +++ b/regression-tests.dnsdist/test_Basics.py @@ -8,7 +8,7 @@ class TestBasics(DNSDistTest): _config_template = """ newServer{address="127.0.0.1:%s"} truncateTC(true) - addAction(AndRule{QTypeRule(DNSQType.ANY), TCPRule(false)}, TCAction()) + addAction(QTypeRule(DNSQType.ANY), TCAction()) addAction(RegexRule("evil[0-9]{4,}\\\\.regex\\\\.tests\\\\.powerdns\\\\.com$"), RCodeAction(DNSRCode.REFUSED)) mySMN = newSuffixMatchNode() mySMN:add(newDNSName("nameAndQtype.tests.powerdns.com.")) diff --git a/regression-tests.dnsdist/test_Caching.py b/regression-tests.dnsdist/test_Caching.py index ac4061afe8a1..587b0f0b3df9 100644 --- a/regression-tests.dnsdist/test_Caching.py +++ b/regression-tests.dnsdist/test_Caching.py @@ -615,6 +615,47 @@ def testCacheSameCookieDifferentECS(self): self.assertEqual(query, receivedQuery) self.assertEqual(receivedResponse, response) +class TestCachingHashingOptions(DNSDistTest): + + _config_template = """ + pc = newPacketCache(100, {maxTTL=86400, minTTL=1, cookieHashing=true, skipOptions={8}}) + getPool(""):setCache(pc) + newServer{address="127.0.0.1:%d"} + """ + + def testCacheDifferentECSSameCookie(self): + """ + Cache: ECS should be ignored by the cache even if cookie is present + """ + ttl = 600 + name = 'cache-different-ecs.cache.tests.powerdns.com.' + eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef') + ecso = clientsubnetoption.ClientSubnetOption('192.0.2.2', 32) + query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso]) + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + ttl, + dns.rdataclass.IN, + dns.rdatatype.AAAA, + '::1') + response.answer.append(rrset) + + # first query to fill the cache + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, response) + + eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef') + ecso = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32) + query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso]) + # second query should be served from the cache + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + receivedResponse.id = response.id + self.assertEqual(receivedResponse, response) + class TestCachingHashingCookies(DNSDistTest): _config_template = """ diff --git a/regression-tests.dnsdist/test_DynBlocks.py b/regression-tests.dnsdist/test_DynBlocks.py index 26801a54e5d4..788a0b17d734 100644 --- a/regression-tests.dnsdist/test_DynBlocks.py +++ b/regression-tests.dnsdist/test_DynBlocks.py @@ -2,6 +2,7 @@ import base64 import json import requests +import socket import time import dns from dnsdisttests import DNSDistTest @@ -15,7 +16,9 @@ class DynBlocksTest(DNSDistTest): _webTimeout = 2.0 _webServerPort = 8083 _webServerBasicAuthPassword = 'secret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' _webServerAPIKey = 'apisecret' + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' def doTestDynBlockViaAPI(self, range, reason, minSeconds, maxSeconds, minBlocks, maxBlocks): headers = {'x-api-key': self._webServerAPIKey} @@ -568,7 +571,7 @@ class TestDynBlockQPS(DynBlocksTest): webserver("127.0.0.1:%s") setWebserverConfig({password="%s", apiKey="%s"}) """ - _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] def testDynBlocksQRate(self): """ @@ -593,7 +596,7 @@ class TestDynBlockGroupQPS(DynBlocksTest): webserver("127.0.0.1:%s") setWebserverConfig({password="%s", apiKey="%s"}) """ - _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] def testDynBlocksQRate(self): """ @@ -777,6 +780,7 @@ def testDynBlocksQRate(self): # check over TCP, which should not be truncated (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response) + receivedQuery.id = query.id self.assertEqual(query, receivedQuery) self.assertEqual(receivedResponse, response) @@ -795,6 +799,7 @@ def testDynBlocksQRate(self): for _ in range((self._dynBlockQPS * self._dynBlockPeriod) + 1): (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response) sent = sent + 1 + receivedQuery.id = query.id self.assertEqual(query, receivedQuery) self.assertEqual(receivedResponse, response) receivedQuery.id = query.id @@ -1149,7 +1154,7 @@ class TestDynBlockGroupNoOp(DynBlocksTest): webserver("127.0.0.1:%s") setWebserverConfig({password="%s", apiKey="%s"}) """ - _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] def testNoOp(self): """ @@ -1213,7 +1218,7 @@ class TestDynBlockGroupWarning(DynBlocksTest): webserver("127.0.0.1:%s") setWebserverConfig({password="%s", apiKey="%s"}) """ - _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_dynBlockWarningQPS', '_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_dynBlockWarningQPS', '_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] def testWarning(self): """ @@ -1261,3 +1266,76 @@ def testWarning(self): self.doTestDynBlockViaAPI('127.0.0.1/32', 'Exceeded query rate', self._dynBlockDuration - 4, self._dynBlockDuration, 0, sent) self.doTestQRate(name) + +class TestDynBlockGroupPort(DNSDistTest): + + _dynBlockQPS = 20 + _dynBlockPeriod = 2 + _dynBlockDuration = 5 + _config_template = """ + local dbr = dynBlockRulesGroup() + dbr:setQueryRate(%d, %d, "Exceeded query rate", %d, DNSAction.Drop) + -- take the exact port into account + dbr:setMasks(32, 128, 16) + + function maintenance() + dbr:apply() + end + newServer{address="127.0.0.1:%d"} + """ + _config_params = ['_dynBlockQPS', '_dynBlockPeriod', '_dynBlockDuration', '_testServerPort'] + + def testPort(self): + """ + Dyn Blocks (group): Exact port matching + """ + name = 'port.group.dynblocks.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '192.0.2.1') + response.answer.append(rrset) + + allowed = 0 + sent = 0 + for _ in range((self._dynBlockQPS * self._dynBlockPeriod) + 1): + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response) + sent = sent + 1 + if receivedQuery: + receivedQuery.id = query.id + self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) + allowed = allowed + 1 + else: + # the query has not reached the responder, + # let's clear the response queue + self.clearToResponderQueue() + + # we might be already blocked, but we should have been able to send + # at least self._dynBlockQPS queries + self.assertGreaterEqual(allowed, self._dynBlockQPS) + + if allowed == sent: + # wait for the maintenance function to run + time.sleep(2) + + # we should now be dropped for up to self._dynBlockDuration + self._dynBlockPeriod + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + + # use a new socket, so a new port + self._toResponderQueue.put(response, True, 1.0) + newsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + newsock.settimeout(1.0) + newsock.connect(("127.0.0.1", self._dnsDistPort)) + newsock.send(query.to_wire()) + receivedResponse = newsock.recv(4096) + if receivedResponse: + receivedResponse = dns.message.from_wire(receivedResponse) + receivedQuery = self._fromResponderQueue.get(True, 1.0) + receivedQuery.id = query.id + self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) diff --git a/regression-tests.dnsdist/test_EDNSSelfGenerated.py b/regression-tests.dnsdist/test_EDNSSelfGenerated.py index 2868f4c97f62..f1ee0aec1ddc 100644 --- a/regression-tests.dnsdist/test_EDNSSelfGenerated.py +++ b/regression-tests.dnsdist/test_EDNSSelfGenerated.py @@ -49,10 +49,8 @@ def testNoEDNS(self): expectedResponse = dns.message.make_response(query) expectedResponse.flags |= dns.flags.TC - for method in ("sendUDPQuery", "sendTCPQuery"): - sender = getattr(self, method) - (_, receivedResponse) = sender(query, response=None, useQueue=False) - self.checkMessageNoEDNS(expectedResponse, receivedResponse) + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.checkMessageNoEDNS(expectedResponse, receivedResponse) name = 'no-edns.lua.edns-self.tests.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN') @@ -104,12 +102,10 @@ def testWithEDNSNoDO(self): expectedResponse = dns.message.make_response(query, our_payload=1042) expectedResponse.flags |= dns.flags.TC - for method in ("sendUDPQuery", "sendTCPQuery"): - sender = getattr(self, method) - (_, receivedResponse) = sender(query, response=None, useQueue=False) - self.checkMessageEDNSWithoutOptions(expectedResponse, receivedResponse) - self.assertFalse(receivedResponse.ednsflags & dns.flags.DO) - self.assertEqual(receivedResponse.payload, 1042) + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.checkMessageEDNSWithoutOptions(expectedResponse, receivedResponse) + self.assertFalse(receivedResponse.ednsflags & dns.flags.DO) + self.assertEqual(receivedResponse.payload, 1042) name = 'edns-no-do.lua.edns-self.tests.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096, want_dnssec=False) @@ -165,12 +161,10 @@ def testWithEDNSWithDO(self): expectedResponse = dns.message.make_response(query, our_payload=1042) expectedResponse.flags |= dns.flags.TC - for method in ("sendUDPQuery", "sendTCPQuery"): - sender = getattr(self, method) - (_, receivedResponse) = sender(query, response=None, useQueue=False) - self.checkMessageEDNSWithoutOptions(expectedResponse, receivedResponse) - self.assertTrue(receivedResponse.ednsflags & dns.flags.DO) - self.assertEqual(receivedResponse.payload, 1042) + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.checkMessageEDNSWithoutOptions(expectedResponse, receivedResponse) + self.assertTrue(receivedResponse.ednsflags & dns.flags.DO) + self.assertEqual(receivedResponse.payload, 1042) name = 'edns-do.lua.edns-self.tests.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096, want_dnssec=True) @@ -227,12 +221,10 @@ def testWithEDNSNoOptions(self): expectedResponse = dns.message.make_response(query, our_payload=1042) expectedResponse.flags |= dns.flags.TC - for method in ("sendUDPQuery", "sendTCPQuery"): - sender = getattr(self, method) - (_, receivedResponse) = sender(query, response=None, useQueue=False) - self.checkMessageEDNSWithoutOptions(expectedResponse, receivedResponse) - self.assertTrue(receivedResponse.ednsflags & dns.flags.DO) - self.assertEqual(receivedResponse.payload, 1042) + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.checkMessageEDNSWithoutOptions(expectedResponse, receivedResponse) + self.assertTrue(receivedResponse.ednsflags & dns.flags.DO) + self.assertEqual(receivedResponse.payload, 1042) name = 'edns-options.lua.edns-self.tests.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN', use_edns=True, options=[ecso], payload=512, want_dnssec=True) @@ -312,10 +304,8 @@ def testWithEDNSNoDO(self): expectedResponse = dns.message.make_response(query) expectedResponse.flags |= dns.flags.TC - for method in ("sendUDPQuery", "sendTCPQuery"): - sender = getattr(self, method) - (_, receivedResponse) = sender(query, response=None, useQueue=False) - self.checkMessageNoEDNS(expectedResponse, receivedResponse) + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.checkMessageNoEDNS(expectedResponse, receivedResponse) name = 'edns-no-do.lua.edns-self-disabled.tests.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096, want_dnssec=False) diff --git a/regression-tests.dnsdist/test_OutgoingDOH.py b/regression-tests.dnsdist/test_OutgoingDOH.py new file mode 100644 index 000000000000..217c885fef0e --- /dev/null +++ b/regression-tests.dnsdist/test_OutgoingDOH.py @@ -0,0 +1,626 @@ +#!/usr/bin/env python +import base64 +import copy +import dns +import requests +import ssl +import threading +import time + +from dnsdisttests import DNSDistTest + +class OutgoingDOHTests(object): + + _webTimeout = 2.0 + _webServerPort = 8083 + _webServerBasicAuthPassword = 'secret' + _webServerAPIKey = 'apisecret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' + + def checkOnlyDOHResponderHit(self, numberOfDOHQueries=1): + self.assertNotIn('UDP Responder', self._responsesCounter) + self.assertNotIn('TCP Responder', self._responsesCounter) + self.assertNotIn('TLS Responder', self._responsesCounter) + self.assertEqual(self._responsesCounter['DoH Connection Handler'], numberOfDOHQueries) + + def getServerStat(self, key): + headers = {'x-api-key': self._webServerAPIKey} + url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost' + r = requests.get(url, headers=headers, timeout=self._webTimeout) + self.assertTrue(r) + self.assertEqual(r.status_code, 200) + self.assertTrue(r.json()) + content = r.json() + self.assertTrue(len(content['servers']), 1) + server = content['servers'][0] + self.assertIn(key, server) + return server[key] + + def testUDP(self): + """ + Outgoing DOH: UDP query is sent via DOH + """ + name = 'udp.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + expectedResponse.answer.append(rrset) + + connsBefore = self.getServerStat('tcpReusedConnections') + + numberOfUDPQueries = 10 + for _ in range(numberOfUDPQueries): + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, expectedResponse) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) + + # there was one TCP query in testTCP (below, but before in alphabetical order) + numberOfQueries = numberOfUDPQueries + 1 + self.checkOnlyDOHResponderHit(numberOfUDPQueries) + + self.assertEqual(self.getServerStat('tcpNewConnections'), 1) + self.assertEqual(self.getServerStat('tcpReusedConnections'), connsBefore + numberOfQueries - 1) + self.assertEqual(self.getServerStat('tlsResumptions'), 0) + + def testTCP(self): + """ + Outgoing DOH: TCP query is sent via DOH + """ + name = 'tcp.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + expectedResponse.answer.append(rrset) + + connsBefore = self.getServerStat('tcpReusedConnections') + + (receivedQuery, receivedResponse) = self.sendTCPQuery(query, expectedResponse) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) + self.checkOnlyDOHResponderHit() + self.assertEqual(self.getServerStat('tcpNewConnections'), 1) + self.assertEqual(self.getServerStat('tcpReusedConnections'), connsBefore) + self.assertEqual(self.getServerStat('tlsResumptions'), 0) + + def testUDPCache(self): + """ + Outgoing DOH: UDP query is sent via DOH, should be cached + """ + name = 'udp.cached.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + expectedResponse.answer.append(rrset) + + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, expectedResponse) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) + + numberOfUDPQueries = 10 + for _ in range(numberOfUDPQueries): + (_, receivedResponse) = self.sendUDPQuery(query, useQueue=False, response=None) + self.assertEqual(receivedResponse, expectedResponse) + + def testTCPCache(self): + """ + Outgoing DOH: TCP query is sent via DOH, should be cached + """ + name = 'tcp.cached.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + expectedResponse.answer.append(rrset) + + (receivedQuery, receivedResponse) = self.sendTCPQuery(query, expectedResponse) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) + + numberOfTCPQueries = 10 + for _ in range(numberOfTCPQueries): + (_, receivedResponse) = self.sendTCPQuery(query, useQueue=False, response=None) + self.assertEqual(receivedResponse, expectedResponse) + + def testZHealthChecks(self): + # this test has to run last, as it will mess up the TCP connection counter, + # hence the 'Z' in the name + self.sendConsoleCommand("getServer(0):setAuto()") + time.sleep(2) + status = self.sendConsoleCommand("if getServer(0):isUp() then return 'up' else return 'down' end").strip("\n") + self.assertEqual(status, 'up') + +class BrokenOutgoingDOHTests(object): + + _webTimeout = 2.0 + _webServerPort = 8083 + _webServerBasicAuthPassword = 'secret' + _webServerAPIKey = 'apisecret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' + + def checkNoResponderHit(self): + self.assertNotIn('UDP Responder', self._responsesCounter) + self.assertNotIn('TCP Responder', self._responsesCounter) + self.assertNotIn('TLS Responder', self._responsesCounter) + self.assertNotIn('DOH Responder', self._responsesCounter) + + def testUDP(self): + """ + Outgoing DOH (broken): UDP query is sent via DOH + """ + name = 'udp.broken-outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + self.checkNoResponderHit() + + def testTCP(self): + """ + Outgoing DOH (broken): TCP query is sent via DOH + """ + name = 'tcp.broken-outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + expectedResponse.answer.append(rrset) + + (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + self.checkNoResponderHit() + +class OutgoingDOHBrokenResponsesTests(object): + + _webTimeout = 2.0 + _webServerPort = 8083 + _webServerBasicAuthPassword = 'secret' + _webServerAPIKey = 'apisecret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' + + def testUDP(self): + """ + Outgoing DOH (broken responses): UDP query is sent via DOH + """ + name = '500-status.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + + name = 'invalid-dns-payload.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + + name = 'closing-connection-id.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + + # but a valid response should be successful + name = 'valid.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + response = dns.message.make_response(query) + + (_, receivedResponse) = self.sendUDPQuery(query, response) + # we can't check the received query because the responder does not populate the queue.. + # self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) + + def testTCP(self): + """ + Outgoing DOH (broken responses): TCP query is sent via DOH + """ + name = '500-status.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + + name = 'invalid-dns-payload.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + + name = 'closing-connection-id.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + + (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, None) + + # but a valid response should be successful + name = 'valid.broken-responses.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + response = dns.message.make_response(query) + + (_, receivedResponse) = self.sendTCPQuery(query, response) + # we can't check the received query because the responder does not populate the queue.. + #self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) + +class TestOutgoingDOHOpenSSL(DNSDistTest, OutgoingDOHTests): + _tlsBackendPort = 10543 + _tlsProvider = 'openssl' + _consoleKey = DNSDistTest.generateConsoleKey() + _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') + _config_params = ['_consoleKeyB64', '_consolePort', '_tlsBackendPort', '_tlsProvider', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setKey("%s") + controlSocket("127.0.0.1:%d") + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='%s', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com', dohPath='/dns-query', pool={'', 'cache'}}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + + pc = newPacketCache(100) + getPool('cache'):setCache(pc) + smn = newSuffixMatchNode() + smn:add('cached.outgoing-doh.test.powerdns.com.') + addAction(SuffixMatchNodeRule(smn), PoolAction('cache')) + """ + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.set_alpn_protocols(["h2"]) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHGnuTLS(DNSDistTest, OutgoingDOHTests): + _tlsBackendPort = 10544 + _tlsProvider = 'gnutls' + _consoleKey = DNSDistTest.generateConsoleKey() + _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') + _config_params = ['_consoleKeyB64', '_consolePort', '_tlsBackendPort', '_tlsProvider', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setKey("%s") + controlSocket("127.0.0.1:%d") + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='%s', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com', dohPath='/dns-query', pool={'', 'cache'}}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + + pc = newPacketCache(100) + getPool('cache'):setCache(pc) + smn = newSuffixMatchNode() + smn:add('cached.outgoing-doh.test.powerdns.com.') + addAction(SuffixMatchNodeRule(smn), PoolAction('cache')) + """ + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.load_cert_chain('server.chain', 'server.key') + tlsContext.keylog_filename = "/tmp/keys" + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHOpenSSLWrongCertName(DNSDistTest, BrokenOutgoingDOHTests): + _tlsBackendPort = 10545 + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='openssl', validateCertificates=true, caStore='ca.pem', subjectName='not-powerdns.com', dohPath='/dns-query'}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + """ + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHGnuTLSWrongCertName(DNSDistTest, BrokenOutgoingDOHTests): + _tlsBackendPort = 10546 + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='gnutls', validateCertificates=true, caStore='ca.pem', subjectName='not-powerdns.com', dohPath='/dns-query'}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + """ + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHOpenSSLWrongCertNameButNoCheck(DNSDistTest, OutgoingDOHTests): + _tlsBackendPort = 10547 + _tlsProvider = 'openssl' + _consoleKey = DNSDistTest.generateConsoleKey() + _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') + _config_params = ['_consoleKeyB64', '_consolePort', '_tlsBackendPort', '_tlsProvider', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setKey("%s") + controlSocket("127.0.0.1:%d") + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='%s', validateCertificates=false, caStore='ca.pem', subjectName='not-powerdns.com', dohPath='/dns-query', pool={'', 'cache'}}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + + pc = newPacketCache(100) + getPool('cache'):setCache(pc) + smn = newSuffixMatchNode() + smn:add('cached.outgoing-doh.test.powerdns.com.') + addAction(SuffixMatchNodeRule(smn), PoolAction('cache')) + """ + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHGnuTLSWrongCertNameButNoCheck(DNSDistTest, OutgoingDOHTests): + _tlsBackendPort = 10548 + _tlsProvider = 'gnutls' + _consoleKey = DNSDistTest.generateConsoleKey() + _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii') + _config_params = ['_consoleKeyB64', '_consolePort', '_tlsBackendPort', '_tlsProvider', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setKey("%s") + controlSocket("127.0.0.1:%d") + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='%s', validateCertificates=false, caStore='ca.pem', subjectName='not-powerdns.com', dohPath='/dns-query', pool={'', 'cache'}}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + + pc = newPacketCache(100) + getPool('cache'):setCache(pc) + smn = newSuffixMatchNode() + smn:add('cached.outgoing-doh.test.powerdns.com.') + addAction(SuffixMatchNodeRule(smn), PoolAction('cache')) + """ + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHBrokenResponsesOpenSSL(DNSDistTest, OutgoingDOHBrokenResponsesTests): + _tlsBackendPort = 10549 + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='openssl', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com', dohPath='/dns-query', pool={'', 'cache'}}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + + pc = newPacketCache(100) + getPool('cache'):setCache(pc) + smn = newSuffixMatchNode() + smn:add('cached.outgoing-doh.test.powerdns.com.') + addAction(SuffixMatchNodeRule(smn), PoolAction('cache')) + """ + + def callback(request, headers, fromQueue, toQueue): + + if str(request.question[0].name) == '500-status.broken-responses.outgoing-doh.test.powerdns.com.': + print("returning 500") + return 500, b'Server error' + + if str(request.question[0].name) == 'invalid-dns-payload.broken-responses.outgoing-doh.test.powerdns.com.': + return 200, b'not DNS' + + if str(request.question[0].name) == 'closing-connection-id.broken-responses.outgoing-doh.test.powerdns.com.': + return 200, None + + print("Returning default for %s" % (request.question[0].name)) + return 200, dns.message.make_response(request).to_wire() + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.set_alpn_protocols(["h2"]) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.callback, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHBrokenResponsesGnuTLS(DNSDistTest, OutgoingDOHBrokenResponsesTests): + _tlsBackendPort = 10550 + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='gnutls', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com', dohPath='/dns-query'}:setUp() + webserver("127.0.0.1:%s") + setWebserverConfig({password="%s", apiKey="%s"}) + """ + _verboseMode = True + + def callback(request, headers, fromQueue, toQueue): + + if str(request.question[0].name) == '500-status.broken-responses.outgoing-doh.test.powerdns.com.': + print("returning 500") + return 500, b'Server error' + + if str(request.question[0].name) == 'invalid-dns-payload.broken-responses.outgoing-doh.test.powerdns.com.': + return 200, b'not DNS' + + if str(request.question[0].name) == 'closing-connection-id.broken-responses.outgoing-doh.test.powerdns.com.': + return 200, None + + print("Returning default for %s" % (request.question[0].name)) + return 200, dns.message.make_response(request).to_wire() + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.set_alpn_protocols(["h2"]) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.callback, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + +class TestOutgoingDOHProxyProtocol(DNSDistTest): + + _tlsBackendPort = 10551 + _config_params = ['_tlsBackendPort'] + _config_template = """ + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='gnutls', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com', dohPath='/dns-query', useProxyProtocol=true}:setUp() + """ + _verboseMode = True + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.set_alpn_protocols(["h2"]) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH woth Proxy Protocol responder..") + cls._DOHResponder = threading.Thread(name='DOH with Proxy Protocol Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext, True]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + + def testPP(self): + """ + Outgoing DOH with Proxy Protocol + """ + name = 'proxy-protocol.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + expectedResponse.answer.append(rrset) + + (receivedProxyPayload, receivedResponse) = self.sendUDPQuery(query, expectedResponse) + receivedQuery = self._fromResponderQueue.get(True, 1.0) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) + self.checkMessageProxyProtocol(receivedProxyPayload, '127.0.0.1', '127.0.0.1', False) + + (receivedProxyPayload, receivedResponse) = self.sendTCPQuery(query, expectedResponse) + receivedQuery = self._fromResponderQueue.get(True, 1.0) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) + self.checkMessageProxyProtocol(receivedProxyPayload, '127.0.0.1', '127.0.0.1', True) + +class TestOutgoingDOHXForwarded(DNSDistTest): + _tlsBackendPort = 10560 + _config_params = ['_tlsBackendPort'] + _config_template = """ + setMaxTCPClientThreads(1) + newServer{address="127.0.0.1:%s", tls='gnutls', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com', dohPath='/dns-query', addXForwardedHeaders=true} + """ + _verboseMode = True + + def callback(request, headersList, fromQueue, toQueue): + + if str(request.question[0].name) == 'a.root-servers.net.': + # do not check headers on health-check queries + return 200, dns.message.make_response(request).to_wire() + + headers = {} + if headersList: + for k,v in headersList: + headers[k] = v + + if not b'x-forwarded-for' in headers: + print("missing X-Forwarded-For") + return 406, b'Missing X-Forwarded-For header' + if not b'x-forwarded-port' in headers: + print("missing X-Forwarded-Port") + return 406, b'Missing X-Forwarded-Port header' + if not b'x-forwarded-proto' in headers: + print("missing X-Forwarded-Proto") + return 406, b'Missing X-Forwarded-Proto header' + + toQueue.put(request, True, 1.0) + response = fromQueue.get(True, 1.0) + if response: + response = copy.copy(response) + response.id = request.id + + return 200, response.to_wire() + + @classmethod + def startResponders(cls): + tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tlsContext.set_alpn_protocols(["h2"]) + tlsContext.load_cert_chain('server.chain', 'server.key') + + print("Launching DOH responder..") + cls._DOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[cls._tlsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.callback, tlsContext]) + cls._DOHResponder.setDaemon(True) + cls._DOHResponder.start() + + def testXForwarded(self): + """ + Outgoing DOH: X-Forwarded + """ + name = 'x-forwarded-for.outgoing-doh.test.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + expectedResponse.answer.append(rrset) + + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, expectedResponse) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) + + (receivedQuery, receivedResponse) = self.sendTCPQuery(query, expectedResponse) + self.assertEqual(query, receivedQuery) + self.assertEqual(receivedResponse, expectedResponse) diff --git a/regression-tests.dnsdist/test_OutgoingTLS.py b/regression-tests.dnsdist/test_OutgoingTLS.py index 6da75473e2f2..53677a66123b 100644 --- a/regression-tests.dnsdist/test_OutgoingTLS.py +++ b/regression-tests.dnsdist/test_OutgoingTLS.py @@ -13,6 +13,8 @@ class OutgoingTLSTests(object): _webServerPort = 8083 _webServerBasicAuthPassword = 'secret' _webServerAPIKey = 'apisecret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' _verboseMode = True def checkOnlyTLSResponderHit(self, numberOfTLSQueries=1): @@ -50,6 +52,7 @@ def testUDP(self): numberOfUDPQueries = 10 for _ in range(numberOfUDPQueries): (receivedQuery, receivedResponse) = self.sendUDPQuery(query, expectedResponse) + receivedQuery.id = query.id self.assertEqual(query, receivedQuery) self.assertEqual(receivedResponse, expectedResponse) @@ -80,6 +83,7 @@ def testTCP(self): expectedResponse.answer.append(rrset) (receivedQuery, receivedResponse) = self.sendTCPQuery(query, expectedResponse) + receivedQuery.id = query.id self.assertEqual(query, receivedQuery) self.assertEqual(receivedResponse, expectedResponse) self.checkOnlyTLSResponderHit() @@ -93,6 +97,8 @@ class BrokenOutgoingTLSTests(object): _webServerPort = 8083 _webServerBasicAuthPassword = 'secret' _webServerAPIKey = 'apisecret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' def checkNoResponderHit(self): self.assertNotIn('UDP Responder', self._responsesCounter) @@ -130,7 +136,7 @@ def testTCP(self): class TestOutgoingTLSOpenSSL(DNSDistTest, OutgoingTLSTests): _tlsBackendPort = 10443 - _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setMaxTCPClientThreads(1) newServer{address="127.0.0.1:%s", tls='openssl', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com'} @@ -150,7 +156,7 @@ def startResponders(cls): class TestOutgoingTLSGnuTLS(DNSDistTest, OutgoingTLSTests): _tlsBackendPort = 10444 - _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setMaxTCPClientThreads(1) newServer{address="127.0.0.1:%s", tls='gnutls', validateCertificates=true, caStore='ca.pem', subjectName='powerdns.com'} @@ -171,7 +177,7 @@ def startResponders(cls): class TestOutgoingTLSOpenSSLWrongCertName(DNSDistTest, BrokenOutgoingTLSTests): _tlsBackendPort = 10445 - _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setMaxTCPClientThreads(1) newServer{address="127.0.0.1:%s", tls='openssl', validateCertificates=true, caStore='ca.pem', subjectName='not-powerdns.com'} @@ -191,7 +197,7 @@ def startResponders(cls): class TestOutgoingTLSGnuTLSWrongCertName(DNSDistTest, BrokenOutgoingTLSTests): _tlsBackendPort = 10446 - _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setMaxTCPClientThreads(1) newServer{address="127.0.0.1:%s", tls='gnutls', validateCertificates=true, caStore='ca.pem', subjectName='not-powerdns.com'} @@ -211,7 +217,7 @@ def startResponders(cls): class TestOutgoingTLSOpenSSLWrongCertNameButNoCheck(DNSDistTest, OutgoingTLSTests): _tlsBackendPort = 10447 - _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setMaxTCPClientThreads(1) newServer{address="127.0.0.1:%s", tls='openssl', validateCertificates=false, caStore='ca.pem', subjectName='not-powerdns.com'} @@ -231,7 +237,7 @@ def startResponders(cls): class TestOutgoingTLSGnuTLSWrongCertNameButNoCheck(DNSDistTest, OutgoingTLSTests): _tlsBackendPort = 10448 - _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _config_params = ['_tlsBackendPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ setMaxTCPClientThreads(1) newServer{address="127.0.0.1:%s", tls='gnutls', validateCertificates=false, caStore='ca.pem', subjectName='not-powerdns.com'} diff --git a/regression-tests.dnsdist/test_Prometheus.py b/regression-tests.dnsdist/test_Prometheus.py index 121764a0933d..51d6f06a0886 100644 --- a/regression-tests.dnsdist/test_Prometheus.py +++ b/regression-tests.dnsdist/test_Prometheus.py @@ -11,8 +11,10 @@ class TestPrometheus(DNSDistTest): _webTimeout = 2.0 _webServerPort = 8083 _webServerBasicAuthPassword = 'secret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' _webServerAPIKey = 'apisecret' - _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' + _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ newServer{address="127.0.0.1:%s"} webserver("127.0.0.1:%s") diff --git a/regression-tests.dnsdist/test_ProxyProtocol.py b/regression-tests.dnsdist/test_ProxyProtocol.py index 18b9efb414e7..bf073f74ed63 100644 --- a/regression-tests.dnsdist/test_ProxyProtocol.py +++ b/regression-tests.dnsdist/test_ProxyProtocol.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import copy import dns import socket import struct @@ -110,7 +111,7 @@ def ProxyProtocolTCPResponder(port, fromQueue, toQueue): toQueue.put([payload, data], True, 2.0) - response = fromQueue.get(True, 2.0) + response = copy.deepcopy(fromQueue.get(True, 2.0)) if not response: conn.close() break @@ -142,36 +143,6 @@ class ProxyProtocolTest(DNSDistTest): _proxyResponderPort = proxyResponderPort _config_params = ['_proxyResponderPort'] - def checkMessageProxyProtocol(self, receivedProxyPayload, source, destination, isTCP, values=[], v6=False, sourcePort=None, destinationPort=None): - proxy = ProxyProtocol() - self.assertTrue(proxy.parseHeader(receivedProxyPayload)) - self.assertEqual(proxy.version, 0x02) - self.assertEqual(proxy.command, 0x01) - if v6: - self.assertEqual(proxy.family, 0x02) - else: - self.assertEqual(proxy.family, 0x01) - if not isTCP: - self.assertEqual(proxy.protocol, 0x02) - else: - self.assertEqual(proxy.protocol, 0x01) - self.assertGreater(proxy.contentLen, 0) - - self.assertTrue(proxy.parseAddressesAndPorts(receivedProxyPayload)) - self.assertEqual(proxy.source, source) - self.assertEqual(proxy.destination, destination) - if sourcePort: - self.assertEqual(proxy.sourcePort, sourcePort) - if destinationPort: - self.assertEqual(proxy.destinationPort, destinationPort) - else: - self.assertEqual(proxy.destinationPort, self._dnsDistPort) - - self.assertTrue(proxy.parseAdditionalValues(receivedProxyPayload)) - proxy.values.sort() - values.sort() - self.assertEqual(proxy.values, values) - class TestProxyProtocol(ProxyProtocolTest): """ dnsdist is configured to prepend a Proxy Protocol header to the query @@ -190,6 +161,7 @@ class TestProxyProtocol(ProxyProtocolTest): addAction("values-action.proxy.tests.powerdns.com.", SetProxyProtocolValuesAction({ ["1"]="dnsdist", ["255"]="proxy-protocol"})) """ _config_params = ['_proxyResponderPort'] + _verboseMode = True def testProxyUDP(self): """ @@ -583,7 +555,6 @@ def testProxyTCPWithValuesFromLua(self): receivedQuery = dns.message.from_wire(receivedDNSData) receivedQuery.id = query.id - receivedResponse.id = response.id self.assertEqual(receivedQuery, query) self.assertEqual(receivedResponse, response) self.checkMessageProxyProtocol(receivedProxyPayload, srcAddr, destAddr, True, [ [0, b'foo'], [1, b'dnsdist'], [ 2, b'foo'], [3, b'proxy'], [ 42, b'bar'], [255, b'proxy-protocol'] ], True, srcPort, destPort) @@ -630,7 +601,6 @@ def testProxyTCPSeveralQueriesOverConnection(self): destPort = 9999 srcAddr = "2001:db8::8" srcPort = 8888 - response = dns.message.make_response(query) tcpPayload = ProxyProtocol.getPayload(False, True, True, srcAddr, destAddr, srcPort, destPort, [ [ 2, b'foo'], [ 3, b'proxy'] ]) @@ -680,7 +650,6 @@ def testProxyTCPSeveralQueriesOverConnection(self): receivedQuery = dns.message.from_wire(receivedDNSData) receivedQuery.id = query.id - receivedResponse.id = response.id self.assertEqual(receivedQuery, query) self.assertEqual(receivedResponse, response) self.checkMessageProxyProtocol(receivedProxyPayload, srcAddr, destAddr, True, [ [0, b'foo'], [1, b'dnsdist'], [ 2, b'foo'], [3, b'proxy'], [ 42, b'bar'], [255, b'proxy-protocol'] ], True, srcPort, destPort) diff --git a/regression-tests.dnsdist/test_SVCB.py b/regression-tests.dnsdist/test_SVCB.py new file mode 100644 index 000000000000..2da43c46e28e --- /dev/null +++ b/regression-tests.dnsdist/test_SVCB.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +import dns +from dnsdisttests import DNSDistTest + +class TestSVCB(DNSDistTest): + + _config_template = """ + local basicSVC = { newSVCRecordParameters(1, "dot.powerdns.com.", { mandatory={"port"}, alpn={"dot"}, noDefaultAlpn=true, port=853, ipv4hint={ "192.0.2.1" }, ipv6hint={ "2001:db8::1" } }), + newSVCRecordParameters(2, "doh.powerdns.com.", { mandatory={"port"}, alpn={"h2"}, port=443, ipv4hint={ "192.0.2.2" }, ipv6hint={ "2001:db8::2" }, key42="/dns-query{?dns}" }) + } + addAction(AndRule{QTypeRule(64), makeRule("basic.svcb.tests.powerdns.com.")}, SpoofSVCAction(basicSVC, {aa=true})) + + local noHintsSVC = { newSVCRecordParameters(1, "dot.powerdns.com.", { mandatory={"port"}, alpn={"dot"}, noDefaultAlpn=true, port=853}), + newSVCRecordParameters(2, "doh.powerdns.com.", { mandatory={"port"}, alpn={"h2"}, port=443, key42="/dns-query{?dns}" }) + } + addAction(AndRule{QTypeRule(64), makeRule("no-hints.svcb.tests.powerdns.com.")}, SpoofSVCAction(noHintsSVC, {aa=true})) + + local effectiveTargetSVC = { newSVCRecordParameters(1, ".", { mandatory={"port"}, alpn={ "dot" }, noDefaultAlpn=true, port=853, ipv4hint={ "192.0.2.1" }, ipv6hint={ "2001:db8::1" }}), + newSVCRecordParameters(2, ".", { mandatory={"port"}, alpn={ "h2" }, port=443, ipv4hint={ "192.0.2.1" }, ipv6hint={ "2001:db8::1" }, key42="/dns-query{?dns}"}) + } + addAction(AndRule{QTypeRule(64), makeRule("effective-target.svcb.tests.powerdns.com.")}, SpoofSVCAction(effectiveTargetSVC, {aa=true})) + + local httpsSVC = { newSVCRecordParameters(1, ".", { mandatory={"port"}, alpn={ "h2" }, noDefaultAlpn=true, port=8002, ipv4hint={ "192.0.2.2" }, ipv6hint={ "2001:db8::2" }}) } + addAction(AndRule{QTypeRule(65), makeRule("https.svcb.tests.powerdns.com.")}, SpoofSVCAction(httpsSVC)) + + newServer{address="127.0.0.1:%s"} + """ + + def testBasic(self): + """ + SVCB: Basic service binding + """ + name = 'basic.svcb.tests.powerdns.com.' + query = dns.message.make_query(name, 64, 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(len(receivedResponse.answer), 1) + self.assertEqual(receivedResponse.answer[0].rdtype, 64) + self.assertEqual(len(receivedResponse.additional), 4) + self.assertEqual(receivedResponse.additional[0], dns.rrset.from_text("doh.powerdns.com.", 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.2')) + self.assertEqual(receivedResponse.additional[1], dns.rrset.from_text("dot.powerdns.com.", 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1')) + self.assertEqual(receivedResponse.additional[2], dns.rrset.from_text("doh.powerdns.com.", 60, dns.rdataclass.IN, dns.rdatatype.AAAA, '2001:db8::2')) + self.assertEqual(receivedResponse.additional[3], dns.rrset.from_text("dot.powerdns.com.", 60, dns.rdataclass.IN, dns.rdatatype.AAAA, '2001:db8::1')) + + def testNoHints(self): + """ + SVCB: No hints + """ + name = 'no-hints.svcb.tests.powerdns.com.' + query = dns.message.make_query(name, 64, 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(len(receivedResponse.answer), 1) + self.assertEqual(receivedResponse.answer[0].rdtype, 64) + self.assertEqual(len(receivedResponse.additional), 0) + + def testEffectiveTarget(self): + """ + SVCB: Effective target + """ + name = 'effective-target.svcb.tests.powerdns.com.' + query = dns.message.make_query(name, 64, 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(len(receivedResponse.answer), 1) + self.assertEqual(receivedResponse.answer[0].rdtype, 64) + self.assertEqual(len(receivedResponse.additional), 2) + self.assertEqual(receivedResponse.additional[0], dns.rrset.from_text(name, 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1')) + self.assertEqual(receivedResponse.additional[1], dns.rrset.from_text(name, 60, dns.rdataclass.IN, dns.rdatatype.AAAA, '2001:db8::1')) + + def testHTTPS(self): + """ + SVCB: HTTPS + """ + name = 'https.svcb.tests.powerdns.com.' + query = dns.message.make_query(name, 65, 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(len(receivedResponse.answer), 1) + self.assertEqual(receivedResponse.answer[0].rdtype, 65) + self.assertEqual(len(receivedResponse.additional), 2) + self.assertEqual(receivedResponse.additional[0], dns.rrset.from_text(name, 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.2')) + self.assertEqual(receivedResponse.additional[1], dns.rrset.from_text(name, 60, dns.rdataclass.IN, dns.rdatatype.AAAA, '2001:db8::2')) diff --git a/regression-tests.dnsdist/test_TCPFastOpen.py b/regression-tests.dnsdist/test_TCPFastOpen.py index 5e050902aea0..73c01a8ba3f5 100644 --- a/regression-tests.dnsdist/test_TCPFastOpen.py +++ b/regression-tests.dnsdist/test_TCPFastOpen.py @@ -18,8 +18,10 @@ class TestBrokenTCPFastOpen(DNSDistTest): _webTimeout = 2.0 _webServerPort = 8083 _webServerBasicAuthPassword = 'secret' + _webServerBasicAuthPasswordHashed = '$scrypt$ln=10,p=1,r=8$6DKLnvUYEeXWh3JNOd3iwg==$kSrhdHaRbZ7R74q3lGBqO1xetgxRxhmWzYJ2Qvfm7JM=' _webServerAPIKey = 'apisecret' - _config_params = ['_testServerPort', '_testServerRetries', '_webServerPort', '_webServerBasicAuthPassword', '_webServerAPIKey'] + _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso=' + _config_params = ['_testServerPort', '_testServerRetries', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] _config_template = """ newServer{address="127.0.0.1:%s", useClientSubnet=true, tcpFastOpen=true, retries=%d } webserver("127.0.0.1:%s") diff --git a/regression-tests.dnsdist/test_TCPOnly.py b/regression-tests.dnsdist/test_TCPOnly.py index 7ada4cacd658..91edd504479c 100644 --- a/regression-tests.dnsdist/test_TCPOnly.py +++ b/regression-tests.dnsdist/test_TCPOnly.py @@ -24,6 +24,7 @@ def testUDP(self): expectedResponse.answer.append(rrset) (receivedQuery, receivedResponse) = self.sendUDPQuery(query, expectedResponse) + receivedQuery.id = query.id self.assertEqual(query, receivedQuery) self.assertEqual(receivedResponse, expectedResponse) @@ -47,6 +48,7 @@ def testTCP(self): expectedResponse.answer.append(rrset) (receivedQuery, receivedResponse) = self.sendTCPQuery(query, expectedResponse) + receivedQuery.id = query.id self.assertEqual(query, receivedQuery) self.assertEqual(receivedResponse, expectedResponse) if 'UDP Responder' in self._responsesCounter: diff --git a/regression-tests.dnsdist/test_Tags.py b/regression-tests.dnsdist/test_Tags.py index a6f7bce0a69f..1e5553a0f03d 100644 --- a/regression-tests.dnsdist/test_Tags.py +++ b/regression-tests.dnsdist/test_Tags.py @@ -187,3 +187,128 @@ def testResponseMatchResponseTagMatches(self): receivedQuery.id = query.id self.assertEqual(query, receivedQuery) self.assertEqual(expectedResponse, receivedResponse) + +class TestSetTagAction(DNSDistTest): + + _config_template = """ + newServer{address="127.0.0.1:%s"} + + addAction(AllRule(), SetTagAction("dns", "value1")) + addAction("tag-me-dns-2.tags.tests.powerdns.com.", SetTagAction("dns", "value2")) + + addAction(TagRule("dns", "value1"), SpoofAction("1.2.3.50")) + addAction(TagRule("dns", "value2"), SpoofAction("1.2.3.4")) + + """ + + def testSetTagDefault(self): + + """ + Tag: Test setTag overwrites existing value + """ + name = 'tag-me-dns-1.tags.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '1.2.3.50') + expectedResponse.answer.append(rrset) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(expectedResponse, receivedResponse) + + def testSetTagOverwritten(self): + + """ + Tag: Test setTag overwrites existing value + """ + name = 'tag-me-dns-2.tags.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '1.2.3.4') + expectedResponse.answer.append(rrset) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(expectedResponse, receivedResponse) + +class TestSetTag(DNSDistTest): + + _config_template = """ + newServer{address="127.0.0.1:%s"} + + function dqset(dq) + dq:setTag("dns", "value1") + if tostring(dq.qname) == 'tag-me-dns-2.tags.tests.powerdns.com.' then + dq:setTag("dns", "value2") + end + return DNSAction.None, "" + end + + addAction(AllRule(), LuaAction(dqset)) + + addAction(TagRule("dns", "value1"), SpoofAction("1.2.3.50")) + addAction(TagRule("dns", "value2"), SpoofAction("1.2.3.4")) + + """ + + def testSetTagDefault(self): + + """ + Tag: Test setTag overwrites existing value + """ + name = 'tag-me-dns-1.tags.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '1.2.3.50') + expectedResponse.answer.append(rrset) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(expectedResponse, receivedResponse) + + def testSetTagOverwritten(self): + + """ + Tag: Test setTag overwrites existing value + """ + name = 'tag-me-dns-2.tags.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + expectedResponse = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '1.2.3.4') + expectedResponse.answer.append(rrset) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertTrue(receivedResponse) + self.assertEqual(expectedResponse, receivedResponse) diff --git a/regression-tests.ixfrdist/requirements.txt b/regression-tests.ixfrdist/requirements.txt index df4d6ee21dcd..645f9614b694 100644 --- a/regression-tests.ixfrdist/requirements.txt +++ b/regression-tests.ixfrdist/requirements.txt @@ -1,4 +1,4 @@ -dnspython -nose +dnspython==2.1.0 +nose>=1.3.7 git+https://github.com/PowerDNS/xfrserver.git@0.1 requests diff --git a/regression-tests.nobackend/counters/command b/regression-tests.nobackend/counters/command index 7f77e457fe3f..a88d4a46e46a 100755 --- a/regression-tests.nobackend/counters/command +++ b/regression-tests.nobackend/counters/command @@ -9,34 +9,34 @@ port=5600 rm -f pdns*.pid $PDNS --daemon=no --local-address=127.0.0.1,::1 \ - --local-port=$port --socket-dir=./ --no-shuffle --launch=random --no-config \ - --module-dir=../regression-tests/modules --zone-cache-refresh-interval=0 & + --local-port=$port --socket-dir=./ --no-shuffle --launch=bind --no-config \ + --module-dir=../regression-tests/modules --bind-config=counters/named.conf & sleep 2 -$SDIG 127.0.0.1 $port random.example.com A >&2 >/dev/null -$SDIG 127.0.0.1 $port example.com SOA >&2 >/dev/null +$SDIG 127.0.0.1 $port server1.test.com A >&2 >/dev/null +$SDIG 127.0.0.1 $port test.com SOA >&2 >/dev/null -$SDIG 127.0.0.1 $port random.example.com A tcp >&2 >/dev/null -$SDIG 127.0.0.1 $port example.com SOA tcp >&2 >/dev/null +$SDIG 127.0.0.1 $port server1.test.com A tcp >&2 >/dev/null +$SDIG 127.0.0.1 $port test.com SOA tcp >&2 >/dev/null -$SDIG ::1 $port random.example.com A >&2 >/dev/null -$SDIG ::1 $port random.example.com A tcp >&2 >/dev/null +$SDIG ::1 $port server1.test.com A >&2 >/dev/null +$SDIG ::1 $port server1.com A tcp >&2 >/dev/null -$SDIG ::1 $port example.com SOA >&2 >/dev/null -$SDIG ::1 $port example.com SOA tcp >&2 >/dev/null +$SDIG ::1 $port test.com SOA >&2 >/dev/null +$SDIG ::1 $port test.com SOA tcp >&2 >/dev/null # NXDOMAIN -$SDIG 127.0.0.1 $port nx.example.com A >&2 >/dev/null +$SDIG 127.0.0.1 $port nx.test.com A >&2 >/dev/null # NOERROR -$SDIG 127.0.0.1 $port example.com SRV >&2 >/dev/null +$SDIG 127.0.0.1 $port test.com SRV >&2 >/dev/null # unauth -$SDIG 127.0.0.1 $port example.invalid MX >&2 >/dev/null +$SDIG 127.0.0.1 $port test.com.invalid MX >&2 >/dev/null $PDNSCONTROL --config-name= --no-config --socket-dir=./ 'show *' | \ - tr ',' '\n'| grep -v -E '(user-msec|sys-msec|cpu-iowait|cpu-steal|uptime|udp-noport-errors|udp-in-errors|real-memory-usage|special-memory-usage|udp-recvbuf-errors|udp-sndbuf-errors|-hit|-miss|fd-usage|latency|backend-queries)' | LC_ALL=C sort + tr ',' '\n'| grep -v -E '(user-msec|sys-msec|cpu-iowait|cpu-steal|uptime|udp-noport-errors|udp-in-csum-errors|udp-in-errors|udp-recvbuf-errors|udp-sndbuf-errors|real-memory-usage|special-memory-usage|udp6-recvbuf-errors|udp6-sndbuf-errors|udp6-noport-errors|udp6-in-csum-errors|udp6-in-errors|-hit|-miss|fd-usage|latency|backend-queries)' | LC_ALL=C sort kill $(cat pdns*.pid) rm pdns*.pid diff --git a/regression-tests.nobackend/counters/expected_result b/regression-tests.nobackend/counters/expected_result index 6ae3c5e88ac1..3d239e72cfb1 100644 --- a/regression-tests.nobackend/counters/expected_result +++ b/regression-tests.nobackend/counters/expected_result @@ -15,9 +15,9 @@ noerror-packets=1 nxdomain-packets=1 open-tcp-connections=0 overload-drops=0 -packetcache-size=7 +packetcache-size=8 qsize-q=0 -query-cache-size=10 +query-cache-size=4 rd-queries=0 recursing-answers=0 recursing-questions=0 @@ -44,26 +44,28 @@ security-status=0 servfail-packets=0 signature-cache-size=0 signatures=0 -tcp-answers-bytes=264 +tcp-answers-bytes=235 tcp-answers=4 +tcp-cookie-queries=0 tcp-queries=4 -tcp4-answers-bytes=132 +tcp4-answers-bytes=128 tcp4-answers=2 tcp4-queries=2 -tcp6-answers-bytes=132 +tcp6-answers-bytes=107 tcp6-answers=2 tcp6-queries=2 timedout-packets=0 -udp-answers-bytes=460 +udp-answers-bytes=449 udp-answers=7 +udp-cookie-queries=0 udp-do-queries=0 udp-queries=7 -udp4-answers-bytes=328 +udp4-answers-bytes=321 udp4-answers=5 udp4-queries=5 -udp6-answers-bytes=132 +udp6-answers-bytes=128 udp6-answers=2 udp6-queries=2 -unauth-packets=1 +unauth-packets=2 xfr-queue=0 -zone-cache-size=0 +zone-cache-size=1 diff --git a/regression-tests.nobackend/counters/named.conf b/regression-tests.nobackend/counters/named.conf new file mode 100644 index 000000000000..110e91523c71 --- /dev/null +++ b/regression-tests.nobackend/counters/named.conf @@ -0,0 +1,14 @@ +options { + directory "../regression-tests/zones/"; + recursion no; + listen-on port 5300 { + 127.0.0.1; + }; + version "Meow!Meow!"; + minimal-responses yes; +}; + +zone "test.com"{ + type master; + file "./test.com"; +}; diff --git a/regression-tests.nobackend/runtests b/regression-tests.nobackend/runtests index fb6cd9cb4507..ef50aae6cdb2 100755 --- a/regression-tests.nobackend/runtests +++ b/regression-tests.nobackend/runtests @@ -1,6 +1,7 @@ #!/bin/sh set -ex cp -f ../regression-tests/zones/test.dyndns.orig ../regression-tests/zones/test.dyndns +cp -f ../regression-tests/zones/sub.test.dyndns.orig ../regression-tests/zones/sub.test.dyndns export testsdir=$(pwd) ../regression-tests/runtests $1 ../regression-tests/toxml @@ -11,7 +12,7 @@ then exit 0; else for t in `cat failed_tests`; do - cat $t/diff + cat $t/diff || true done exit 1; diff --git a/regression-tests.recursor-dnssec/recursortests.py b/regression-tests.recursor-dnssec/recursortests.py index 8cbdba1d260a..f2602a23cd52 100644 --- a/regression-tests.recursor-dnssec/recursortests.py +++ b/regression-tests.recursor-dnssec/recursortests.py @@ -526,11 +526,11 @@ def startAuth(cls, confdir, ipaddress): print("Launching pdns_server..") authcmd = list(cls._auth_cmd) authcmd.append('--config-dir=%s' % confdir) - authcmd.append('--local-address=%s' % ipaddress) - if (confdir[-4:] == "ROOT") and have_ipv6(): - authcmd.append('--local-ipv6=::1') - else: - authcmd.append('--local-ipv6=') + ipconfig = ipaddress + # auth-8 is the auth serving the root, it gets an ipv6 address + if (confdir[-6:] == "auth-8") and have_ipv6(): + ipconfig += ',::1' + authcmd.append('--local-address=%s' % ipconfig) print(' '.join(authcmd)) logFile = os.path.join(confdir, 'pdns.log') diff --git a/regression-tests.recursor-dnssec/runtests b/regression-tests.recursor-dnssec/runtests index 08b0e5efad80..600d681adf23 100755 --- a/regression-tests.recursor-dnssec/runtests +++ b/regression-tests.recursor-dnssec/runtests @@ -55,6 +55,12 @@ if ! "$PDNSRECURSOR" --version 2>&1 | grep Features | grep -q dnstap-framestream export NODNSTAPTESTS=1 fi +# libfstrm has a bad interaction with libfaketime on at least Ubuntu Focal. +# to run the test without LIBFAKETIME, we clear the var if it set to /bin/false +if [ "$LIBFAKETIME" = "/bin/false" ]; then + LIBFAKETIME="" +fi + if [ "${LIBAUTHBIND}" != "" -o "${LIBFAKETIME}" != "" ]; then LD_PRELOAD="${LIBASAN} ${LIBAUTHBIND} ${LIBFAKETIME}" nosetests -I test_WellKnown.py --with-xunit $@ else diff --git a/regression-tests.recursor-dnssec/test_Carbon.py b/regression-tests.recursor-dnssec/test_Carbon.py new file mode 100644 index 000000000000..83a70d7acfaf --- /dev/null +++ b/regression-tests.recursor-dnssec/test_Carbon.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +import threading +import socket +import sys +import time +import os +from queue import Queue + +from recursortests import RecursorTest + +class TestCarbon(RecursorTest): + _confdir = 'Carbon' + _carbonNamespace = 'NS' + _carbonInstance = 'Instance' + _carbonServerName = "carbonname1" + _carbonInterval = 2 + _carbonServer1Port = 8000 + _carbonServer2Port = 8001 + _carbonQueue1 = Queue() + _carbonQueue2 = Queue() + _carbonCounters = {} + _config_template = """ + carbon-namespace=%s + carbon-instance=%s + carbon-interval=%s + carbon-ourname=%s + carbon-server=127.0.0.1:%s,127.0.01:%s + """ % (_carbonNamespace, _carbonInstance, _carbonInterval, _carbonServerName, _carbonServer1Port, _carbonServer2Port) + @classmethod + def setUpClass(cls): + + # we don't need all the auth stuff + cls.setUpSockets() + cls.startResponders() + + confdir = os.path.join('configs', cls._confdir) + cls.createConfigDir(confdir) + + cls.generateRecursorConfig(confdir) + cls.startRecursor(confdir, cls._recursorPort) + + @classmethod + def tearDownClass(cls): + cls.tearDownRecursor() + + @classmethod + def CarbonResponder(cls, port): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + try: + sock.bind(("127.0.0.1", port)) + except socket.error as e: + print("Error binding in the Carbon responder: %s" % str(e)) + sys.exit(1) + + sock.listen(100) + while True: + (conn, _) = sock.accept() + conn.settimeout(2.0) + lines = b'' + while True: + data = conn.recv(4096) + if not data: + break + lines += data + + if port == cls._carbonServer1Port: + cls._carbonQueue1.put(lines, True, timeout=2.0) + else: + cls._carbonQueue2.put(lines, True, timeout=2.0) + if threading.currentThread().name in cls._carbonCounters: + cls._carbonCounters[threading.currentThread().name] += 1 + else: + cls._carbonCounters[threading.currentThread().name] = 1 + + conn.close() + sock.close() + + @classmethod + def startResponders(cls): + cls._CarbonResponder1 = threading.Thread(name='Carbon Responder 1', target=cls.CarbonResponder, args=[cls._carbonServer1Port]) + cls._CarbonResponder1.setDaemon(True) + cls._CarbonResponder1.start() + + cls._CarbonResponder2 = threading.Thread(name='Carbon Responder 2', target=cls.CarbonResponder, args=[cls._carbonServer2Port]) + cls._CarbonResponder2.setDaemon(True) + cls._CarbonResponder2.start() + + def testCarbon(self): + """ + Carbon: send data to 2 carbon servers + """ + # wait for the carbon data to be sent + time.sleep(self._carbonInterval + 1) + + # check if the servers have received our data + # we will block for a short while if the data is not already there, + # and an exception will be raised after the timeout + # first server + data1 = self._carbonQueue1.get(block=True, timeout=2.0) + # second server + data2 = self._carbonQueue2.get(block=True, timeout=2.0) + after = time.time() + + self.assertTrue(data1) + self.assertTrue(len(data1.splitlines()) > 1) + expectedStart = b"%s.%s.%s." % (self._carbonNamespace.encode('UTF8'), self._carbonServerName.encode('UTF-8'), self._carbonInstance.encode('UTF8')) + for line in data1.splitlines(): + self.assertTrue(line.startswith(expectedStart)) + parts = line.split(b' ') + self.assertEqual(len(parts), 3) + self.assertTrue(parts[1].isdigit()) + self.assertTrue(parts[2].isdigit()) + self.assertTrue(int(parts[2]) <= int(after)) + + self.assertTrue(data2) + self.assertTrue(len(data2.splitlines()) > 1) + expectedStart = b"%s.%s.%s." % (self._carbonNamespace.encode('UTF8'), self._carbonServerName.encode('UTF-8'), self._carbonInstance.encode('UTF8')) + for line in data2.splitlines(): + self.assertTrue(line.startswith(expectedStart)) + parts = line.split(b' ') + self.assertEqual(len(parts), 3) + self.assertTrue(parts[1].isdigit()) + self.assertTrue(parts[2].isdigit()) + self.assertTrue(int(parts[2]) <= int(after)) + + # make sure every carbon server has received at least one connection + for key in self._carbonCounters: + value = self._carbonCounters[key] + self.assertTrue(value >= 1) + diff --git a/regression-tests.recursor-dnssec/test_Expired.py b/regression-tests.recursor-dnssec/test_Expired.py index 7ef785d1d549..14c4948ecdc1 100644 --- a/regression-tests.recursor-dnssec/test_Expired.py +++ b/regression-tests.recursor-dnssec/test_Expired.py @@ -4,6 +4,7 @@ import time import dns +import extendederrors from recursortests import RecursorTest @@ -24,3 +25,31 @@ def testA(self): res = self.sendUDPQuery(query) self.assertRcodeEqual(res, dns.rcode.SERVFAIL) + +class testExpiredWithEDE(RecursorTest): + """This regression test starts the authoritative servers with a clock that is + set 15 days into the past. Hence, the recursor must reject the signatures + because they are expired. + """ + _confdir = 'ExpiredWithEDE' + + _config_template = """ + dnssec=validate + extended-resolution-errors=yes + """ + + _auth_env = {'LD_PRELOAD':os.environ.get('LIBFAKETIME'), + 'FAKETIME':'-15d'} + + def testA(self): + qname = 'host1.secure.example' + query = dns.message.make_query(qname, 'A', want_dnssec=True) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + res = sender(query, timeout=5.0) + self.assertRcodeEqual(res, dns.rcode.SERVFAIL) + self.assertEqual(res.edns, 0) + self.assertEqual(len(res.options), 1) + self.assertEqual(res.options[0].otype, 15) + self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(7, b'')) diff --git a/regression-tests.recursor-dnssec/test_ExtendedErrors.py b/regression-tests.recursor-dnssec/test_ExtendedErrors.py index 0d1a6851abaa..855d3f11196b 100644 --- a/regression-tests.recursor-dnssec/test_ExtendedErrors.py +++ b/regression-tests.recursor-dnssec/test_ExtendedErrors.py @@ -119,6 +119,19 @@ def testExpired(self): self.assertEqual(res.options[0].otype, 15) self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(7, b'')) + def testAllExpired(self): + qname = 'servfail.nl.' + query = dns.message.make_query(qname, 'AAAA', want_dnssec=True) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + res = sender(query, timeout=5.0) + self.assertRcodeEqual(res, dns.rcode.SERVFAIL) + self.assertEqual(res.edns, 0) + self.assertEqual(len(res.options), 1) + self.assertEqual(res.options[0].otype, 15) + self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(6, b'')) + def testBogus(self): qname = 'bogussig.ok.bad-dnssec.wb.sidnlabs.nl.' query = dns.message.make_query(qname, 'A', want_dnssec=True) diff --git a/regression-tests.recursor-dnssec/test_Lua.py b/regression-tests.recursor-dnssec/test_Lua.py index 6053fbb9938b..bd0c00ba653b 100644 --- a/regression-tests.recursor-dnssec/test_Lua.py +++ b/regression-tests.recursor-dnssec/test_Lua.py @@ -714,7 +714,7 @@ def testValidationBogus(self): self.assertEqual(len(res.authority), 0) class PolicyEventFilterOnFollowUpTest(RecursorTest): - """Tests the interaction between RPZ and followup queries (dns64, folliwCNAME) + """Tests the interaction between RPZ and followup queries (dns64, followCNAME) """ _confdir = 'policyeventfilter-followup' @@ -763,3 +763,41 @@ def testA(self): self.assertEqual(len(res.answer), 2) self.assertEqual(len(res.authority), 0) self.assertResponseMatches(query, expected, res) + +class PolicyEventFilterOnFollowUpWithNativeDNS64Test(RecursorTest): + """Tests the interaction between followup queries and native dns64 + """ + + _confdir = 'policyeventfilter-followup-dns64' + _config_template = """ + dns64-prefix=1234::/96 + """ + _lua_config_file = """ + """ + _lua_dns_script_file = """ + function preresolve(dq) + dq:addAnswer(pdns.CNAME, "cname.secure.example.") + dq.followupFunction="followCNAMERecords" + dq.rcode = pdns.NOERROR + return true + end + + """ + + def testAAAA(self): + expected = [ + dns.rrset.from_text('mx1.secure.example.', 15, dns.rdataclass.IN, 'CNAME', 'cname.secure.example.'), + dns.rrset.from_text('cname.secure.example.', 15, dns.rdataclass.IN, 'CNAME', ' host1.secure.example.'), + dns.rrset.from_text('host1.secure.example.', 15, dns.rdataclass.IN, 'AAAA', '1234::c000:202') + ] + query = dns.message.make_query('mx1.secure.example', 'AAAA') + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + res = sender(query) + print(res) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertEqual(len(res.answer), 3) + self.assertEqual(len(res.authority), 0) + self.assertResponseMatches(query, expected, res) + diff --git a/regression-tests.recursor-dnssec/test_Notify.py b/regression-tests.recursor-dnssec/test_Notify.py new file mode 100644 index 000000000000..6ff0ea346ff6 --- /dev/null +++ b/regression-tests.recursor-dnssec/test_Notify.py @@ -0,0 +1,117 @@ +import clientsubnetoption +import cookiesoption +import dns +import os +import requests +import subprocess + +from recursortests import RecursorTest + +class NotifyRecursorTest(RecursorTest): + + _auth_zones = { + '8': {'threads': 1, + 'zones': ['ROOT']} + } + + _confdir = 'Notify' + _wsPort = 8042 + _wsTimeout = 2 + _wsPassword = 'secretpassword' + _apiKey = 'secretapikey' + _config_template = """ + disable-packetcache=yes + auth-zones=example=configs/%s/example.zone + allow-notify-from=127.0.0.1 + allow-notify-for=example + quiet=no + loglevel=9 + webserver=yes + webserver-port=%d + webserver-address=127.0.0.1 + webserver-password=%s + api-key=%s + """ % (_confdir, _wsPort, _wsPassword, _apiKey) + + @classmethod + def generateRecursorConfig(cls, confdir): + authzonepath = os.path.join(confdir, 'example.zone') + with open(authzonepath, 'w') as authzone: + authzone.write("""$ORIGIN example. +@ 3600 IN SOA {soa} +a 3600 IN A 192.0.2.42 +b 3600 IN A 192.0.2.42 +c 3600 IN A 192.0.2.42 +d 3600 IN A 192.0.2.42 +e 3600 IN A 192.0.2.42 +f 3600 IN CNAME f ; CNAME loop: dirty trick to get a ServFail in an authzone +""".format(soa=cls._SOA)) + super(NotifyRecursorTest, cls).generateRecursorConfig(confdir) + + def checkRecordCacheMetrics(self, expectedHits, expectedMisses): + headers = {'x-api-key': self._apiKey} + url = 'http://127.0.0.1:' + str(self._wsPort) + '/api/v1/servers/localhost/statistics' + r = requests.get(url, headers=headers, timeout=self._wsTimeout) + self.assertTrue(r) + self.assertEqual(r.status_code, 200) + self.assertTrue(r.json()) + content = r.json() + foundHits = False + foundMisses = True + for entry in content: + if entry['name'] == 'cache-hits': + foundHits = True + self.assertEqual(int(entry['value']), expectedHits) + elif entry['name'] == 'cache-misses': + foundMisses = True + self.assertEqual(int(entry['value']), expectedMisses) + + self.assertTrue(foundHits) + self.assertTrue(foundMisses) + + def testNotify(self): + # first query + qname = 'a.example.' + query = dns.message.make_query(qname, 'A', want_dnssec=True) + expected = dns.rrset.from_text(qname, 0, dns.rdataclass.IN, 'A', '192.0.2.42') + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + res = sender(query) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertRRsetInAnswer(res, expected) + + self.checkRecordCacheMetrics(1, 1) + + # we should get a hit over UDP this time + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertRRsetInAnswer(res, expected) + self.checkRecordCacheMetrics(2, 1) + + # we should get a hit over TCP this time + res = self.sendTCPQuery(query) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertRRsetInAnswer(res, expected) + self.checkRecordCacheMetrics(3, 1) + + notify = dns.message.make_query('example', 'SOA', want_dnssec=False) + notify.set_opcode(4) # notify + notifyexpected = dns.rrset.from_text('example.', 0, dns.rdataclass.IN, 'SOA') + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + res = sender(notify) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertEquals(res.opcode(), 4) + print(res) + self.assertEquals(res.question[0].to_text(), 'example. IN SOA') + + self.checkRecordCacheMetrics(3, 1) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + res = sender(query) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.assertRRsetInAnswer(res, expected) + + self.checkRecordCacheMetrics(4, 2) diff --git a/regression-tests.recursor-dnssec/test_PacketCache.py b/regression-tests.recursor-dnssec/test_PacketCache.py index e2be43a2dfd9..35b6039558be 100644 --- a/regression-tests.recursor-dnssec/test_PacketCache.py +++ b/regression-tests.recursor-dnssec/test_PacketCache.py @@ -3,6 +3,7 @@ import dns import os import requests +import subprocess from recursortests import RecursorTest @@ -19,7 +20,8 @@ class PacketCacheRecursorTest(RecursorTest): _wsPassword = 'secretpassword' _apiKey = 'secretapikey' _config_template = """ - packetcache-ttl=60 + packetcache-ttl=10 + packetcache-servfail-ttl=5 auth-zones=example=configs/%s/example.zone webserver=yes webserver-port=%d @@ -39,6 +41,7 @@ def generateRecursorConfig(cls, confdir): c 3600 IN A 192.0.2.42 d 3600 IN A 192.0.2.42 e 3600 IN A 192.0.2.42 +f 3600 IN CNAME f ; CNAME loop: dirty trick to get a ServFail in an authzone """.format(soa=cls._SOA)) super(PacketCacheRecursorTest, cls).generateRecursorConfig(confdir) @@ -136,3 +139,37 @@ def testPacketCache(self): self.assertRcodeEqual(res, dns.rcode.NOERROR) self.assertRRsetInAnswer(res, expected) self.checkPacketCacheMetrics(6, 4) + + # NXDomain should get default packetcache TTL (10) + query = dns.message.make_query('nxdomain.example.', 'A', want_dnssec=True) + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, dns.rcode.NXDOMAIN) + self.checkPacketCacheMetrics(6, 5) + + # NoData should get default packetcache TTL (10) + query = dns.message.make_query('a.example.', 'AAAA', want_dnssec=True) + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, dns.rcode.NOERROR) + self.checkPacketCacheMetrics(6, 6) + + # ServFail should get ServFail TTL (5) + query = dns.message.make_query('f.example.', 'A', want_dnssec=True) + res = self.sendUDPQuery(query) + self.assertRcodeEqual(res, dns.rcode.SERVFAIL) + self.checkPacketCacheMetrics(6, 7) + + # We peek into the cache to check TTLs and allow TTLs to be one lower than inserted since the clock might have ticked + rec_controlCmd = [os.environ['RECCONTROL'], + '--config-dir=%s' % 'configs/' + self._confdir, + 'dump-cache', '-'] + try: + ret = subprocess.check_output(rec_controlCmd, stderr=subprocess.STDOUT) + self.assertTrue((b"a.example. 10 A ; tag 0 udp\n" in ret) or (b"a.example. 9 A ; tag 0 udp\n" in ret)) + self.assertTrue((b"nxdomain.example. 10 A ; tag 0 udp\n" in ret) or (b"nxdomain.example. 9 A ; tag 0 udp\n" in ret)) + self.assertTrue((b"a.example. 10 AAAA ; tag 0 udp\n" in ret) or (b"a.example. 9 AAAA ; tag 0 udp\n" in ret)) + self.assertTrue((b"f.example. 5 A ; tag 0 udp\n" in ret) or (b"f.example. 4 A ; tag 0 udp\n" in ret)) + + except subprocess.CalledProcessError as e: + print(e.output) + raise + diff --git a/regression-tests.recursor-dnssec/test_Prometheus.py b/regression-tests.recursor-dnssec/test_Prometheus.py new file mode 100644 index 000000000000..7c8c599e4b79 --- /dev/null +++ b/regression-tests.recursor-dnssec/test_Prometheus.py @@ -0,0 +1,82 @@ +import os +import requests +import subprocess +import unittest + +from recursortests import RecursorTest + +class RecPrometheusTest(RecursorTest): + + @classmethod + def setUpClass(cls): + + # we don't need all the auth stuff + cls.setUpSockets() + cls.startResponders() + + confdir = os.path.join('configs', cls._confdir) + cls.createConfigDir(confdir) + + cls.generateRecursorConfig(confdir) + cls.startRecursor(confdir, cls._recursorPort) + + @classmethod + def tearDownClass(cls): + cls.tearDownRecursor() + + def checkPrometheusContentBasic(self, content): + for line in content.splitlines(): + if line.startswith('# HELP'): + tokens = line.split(' ') + self.assertGreaterEqual(len(tokens), 4) + elif line.startswith('# TYPE'): + tokens = line.split(' ') + self.assertEqual(len(tokens), 4) + self.assertIn(tokens[3], ['counter', 'gauge', 'histogram']) + elif not line.startswith('#'): + tokens = line.split(' ') + self.assertEqual(len(tokens), 2) + if not line.startswith('pdns_recursor_'): + raise AssertionError('Expecting prometheus metric to be prefixed by \'pdns_recursor_\', got: "%s"' % (line)) + + def checkPrometheusContentPromtool(self, content): + output = None + try: + testcmd = ['promtool', 'check', 'metrics'] + process = subprocess.Popen(testcmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) + output = process.communicate(input=content) + except subprocess.CalledProcessError as exc: + raise AssertionError('%s failed (%d): %s' % (testcmd, process.returncode, process.output)) + + # commented out because promtool returns 3 because of the "_total" suffix warnings + #if process.returncode != 0: + # raise AssertionError('%s failed (%d): %s' % (testcmd, process.returncode, output)) + + for line in output[0].splitlines(): + if line.endswith(b"should have \"_total\" suffix"): + continue + raise AssertionError('%s returned an unexpected output. Faulty line is "%s", complete content is "%s"' % (testcmd, line, output)) + +class BasicPrometheusTest(RecPrometheusTest): + _confdir = 'Prometheus' + _wsPort = 8042 + _wsTimeout = 2 + _wsPassword = 'secretpassword' + _apiKey = 'secretapikey' + + _config_template = """ +webserver=yes +webserver-port=%d +webserver-address=127.0.0.1 +webserver-password=%s +webserver-allow-from=127.0.0.1 +api-key=%s +""" % (_wsPort, _wsPassword, _apiKey) + + def testPrometheus(self): + url = 'http://user:' + self._wsPassword + '@127.0.0.1:' + str(self._wsPort) + '/metrics' + r = requests.get(url, timeout=self._wsTimeout) + self.assertTrue(r) + self.assertEqual(r.status_code, 200) + self.checkPrometheusContentBasic(r.text) + self.checkPrometheusContentPromtool(r.content) diff --git a/regression-tests.recursor-dnssec/test_Protobuf.py b/regression-tests.recursor-dnssec/test_Protobuf.py index 9a6e54fd662c..92f5155102a4 100644 --- a/regression-tests.recursor-dnssec/test_Protobuf.py +++ b/regression-tests.recursor-dnssec/test_Protobuf.py @@ -172,13 +172,13 @@ def checkOutgoingProtobufBase(self, msg, protocol, query, initiator, length=None # compare inBytes with length of query/response self.assertEqual(msg.inBytes, len(query.to_wire())) - def checkProtobufQuery(self, msg, protocol, query, qclass, qtype, qname, initiator='127.0.0.1'): + def checkProtobufQuery(self, msg, protocol, query, qclass, qtype, qname, initiator='127.0.0.1', to='127.0.0.1'): self.assertEqual(msg.type, dnsmessage_pb2.PBDNSMessage.DNSQueryType) self.checkProtobufBase(msg, protocol, query, initiator) # dnsdist doesn't fill the responder field for responses # because it doesn't keep the information around. self.assertTrue(msg.HasField('to')) - self.assertEqual(socket.inet_ntop(socket.AF_INET, msg.to), '127.0.0.1') + self.assertEqual(socket.inet_ntop(socket.AF_INET, msg.to), to) self.assertTrue(msg.HasField('question')) self.assertTrue(msg.question.HasField('qClass')) self.assertEqual(msg.question.qClass, qclass) @@ -368,6 +368,40 @@ def testCNAME(self): self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '192.0.2.42') self.checkNoRemainingMessage() +class ProtobufProxyTest(TestRecursorProtobuf): + """ + This test makes sure that we correctly export addresses over protobuf when the proxy protocol is used. + """ + + _confdir = 'ProtobufProxy' + _config_template = """ +auth-zones=example=configs/%s/example.zone +proxy-protocol-from=127.0.0.1/32 +allow-from=127.0.0.1,6.6.6.6 +""" % _confdir + + def testA(self): + name = 'a.example.' + expected = dns.rrset.from_text(name, 0, dns.rdataclass.IN, 'A', '192.0.2.42') + query = dns.message.make_query(name, 'A', want_dnssec=True) + query.flags |= dns.flags.CD + res = self.sendUDPQueryWithProxyProtocol(query, False, '6.6.6.6', '7.7.7.7', 666, 777) + + self.assertRRsetInAnswer(res, expected) + + # check the protobuf messages corresponding to the UDP query and answer + msg = self.getFirstProtobufMessage() + self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name, '6.6.6.6', '7.7.7.7') + # then the response + msg = self.getFirstProtobufMessage() + self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, res, '6.6.6.6') + self.assertEqual(len(msg.response.rrs), 1) + rr = msg.response.rrs[0] + # we have max-cache-ttl set to 15 + self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, name, 15) + self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '192.0.2.42') + self.checkNoRemainingMessage() + class OutgoingProtobufDefaultTest(TestRecursorProtobuf): """ This test makes sure that we correctly export outgoing queries over protobuf. @@ -905,6 +939,7 @@ def generateRecursorConfig(cls, confdir): authzone.write("""$ORIGIN example. @ 3600 IN SOA {soa} sub.test 3600 IN A 192.0.2.42 +ip 3600 IN A 33.22.11.99 """.format(soa=cls._SOA)) rpzFilePath = os.path.join(confdir, 'zone.rpz') @@ -912,6 +947,7 @@ def generateRecursorConfig(cls, confdir): rpzZone.write("""$ORIGIN zone.rpz. @ 3600 IN SOA {soa} *.test.example.zone.rpz. 60 IN CNAME rpz-passthru. +24.0.11.22.33.rpz-ip 60 IN A 1.2.3.4 """.format(soa=cls._SOA)) super(ProtobufRPZTest, cls).generateRecursorConfig(confdir) @@ -939,6 +975,25 @@ def testA(self): self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '192.0.2.42') self.checkNoRemainingMessage() + def testB(self): + name = 'ip.example.' + expected = dns.rrset.from_text(name, 0, dns.rdataclass.IN, 'A', '1.2.3.4') + query = dns.message.make_query(name, 'A', want_dnssec=True) + query.flags |= dns.flags.CD + res = self.sendUDPQuery(query) + self.assertRRsetInAnswer(res, expected) + + # check the protobuf messages corresponding to the UDP query and answer + msg = self.getFirstProtobufMessage() + self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name) + + # then the response + msg = self.getFirstProtobufMessage() + self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, res) + self.checkProtobufPolicy(msg, dnsmessage_pb2.PBDNSMessage.PolicyType.RESPONSEIP, 'zone.rpz.', '24.0.11.22.33.rpz-ip.', '33.22.11.99', dnsmessage_pb2.PBDNSMessage.PolicyKind.Custom) + self.assertEqual(len(msg.response.rrs), 1) + self.checkNoRemainingMessage() + class ProtobufRPZTagsTest(TestRecursorProtobuf): """ This test makes sure that we correctly export the RPZ tags in our protobuf messages diff --git a/regression-tests.recursor-dnssec/test_RPZIncomplete.py b/regression-tests.recursor-dnssec/test_RPZIncomplete.py new file mode 100644 index 000000000000..906d0f28dadf --- /dev/null +++ b/regression-tests.recursor-dnssec/test_RPZIncomplete.py @@ -0,0 +1,240 @@ +import dns +import json +import os +import requests +import socket +import struct +import sys +import threading +import time + +from recursortests import RecursorTest + +class BadRPZServer(object): + + def __init__(self, port): + self._currentSerial = 0 + self._targetSerial = 1 + self._serverPort = port + listener = threading.Thread(name='RPZ Listener', target=self._listener, args=[]) + listener.setDaemon(True) + listener.start() + + def getCurrentSerial(self): + return self._currentSerial + + def moveToSerial(self, newSerial): + if newSerial == self._currentSerial: + return False + + #if newSerial != self._currentSerial + 1: + # raise AssertionError("Asking the RPZ server to serve serial %d, already serving %d" % (newSerial, self._currentSerial)) + self._targetSerial = newSerial + return True + + def _getAnswer(self, message): + + response = dns.message.make_response(message) + records = [] + + if message.question[0].rdtype == dns.rdatatype.AXFR: + if self._currentSerial != 0: + print('Received an AXFR query but IXFR expected because the current serial is %d' % (self._currentSerial)) + return (None, self._currentSerial) + + newSerial = self._targetSerial + records = [ + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('a.example.zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1'), + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial) + ] + + elif message.question[0].rdtype == dns.rdatatype.IXFR: + oldSerial = message.authority[0][0].serial + + newSerial = self._targetSerial + if newSerial == 2: + records = [ + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % oldSerial), + # no deletion + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('b.example.zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1'), + ] + elif newSerial == 3: + records = [ + dns.rrset.from_text('zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.SOA, 'ns.zone.rpz. hostmaster.zone.rpz. %d 3600 3600 3600 1' % newSerial), + dns.rrset.from_text('a.example.zone.rpz.', 60, dns.rdataclass.IN, dns.rdatatype.A, '192.0.2.1'), + ] + + response.answer = records + return (newSerial, response) + + def _connectionHandler(self, conn): + data = None + while True: + data = conn.recv(2) + if not data: + break + (datalen,) = struct.unpack("!H", data) + data = conn.recv(datalen) + if not data: + break + + message = dns.message.from_wire(data) + if len(message.question) != 1: + print('Invalid RPZ query, qdcount is %d' % (len(message.question)), file=sys.stderr) + break + if not message.question[0].rdtype in [dns.rdatatype.AXFR, dns.rdatatype.IXFR]: + print('Invalid RPZ query, qtype is %d' % (message.question.rdtype), file=sys.stderr) + break + (serial, answer) = self._getAnswer(message) + if not answer: + print('Unable to get a response for %s %d' % (message.question[0].name, message.question[0].rdtype), file=sys.stderr) + break + + wire = answer.to_wire() + conn.send(struct.pack("!H", len(wire))) + conn.send(wire) + self._currentSerial = serial + break + + conn.close() + + def _listener(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + try: + sock.bind(("127.0.0.1", self._serverPort)) + except socket.error as e: + print("Error binding in the RPZ listener: %s" % str(e)) + sys.exit(1) + + sock.listen(100) + while True: + try: + (conn, _) = sock.accept() + thread = threading.Thread(name='RPZ Connection Handler', + target=self._connectionHandler, + args=[conn]) + thread.setDaemon(True) + thread.start() + + except socket.error as e: + print('Error in RPZ socket: %s' % str(e)) + sock.close() + +class RPZIncompleteRecursorTest(RecursorTest): + _wsPort = 8042 + _wsTimeout = 2 + _wsPassword = 'secretpassword' + _apiKey = 'secretapikey' + _confdir = 'RPZIncomplete' + _auth_zones = { + '8': {'threads': 1, + 'zones': ['ROOT']}, + '10': {'threads': 1, + 'zones': ['example']}, + } + + _config_template = """ +auth-zones=example=configs/%s/example.zone +webserver=yes +webserver-port=%d +webserver-address=127.0.0.1 +webserver-password=%s +api-key=%s +log-rpz-changes=yes +""" % (_confdir, _wsPort, _wsPassword, _apiKey) + + def checkRPZStats(self, serial, recordsCount, fullXFRCount, totalXFRCount, failedXFRCount): + headers = {'x-api-key': self._apiKey} + url = 'http://127.0.0.1:' + str(self._wsPort) + '/api/v1/servers/localhost/rpzstatistics' + r = requests.get(url, headers=headers, timeout=self._wsTimeout) + self.assertTrue(r) + self.assertEqual(r.status_code, 200) + self.assertTrue(r.json()) + content = r.json() + self.assertIn('zone.rpz.', content) + zone = content['zone.rpz.'] + for key in ['last_update', 'records', 'serial', 'transfers_failed', 'transfers_full', 'transfers_success']: + self.assertIn(key, zone) + + self.assertEqual(zone['serial'], serial) + self.assertEqual(zone['records'], recordsCount) + self.assertEqual(zone['transfers_full'], fullXFRCount) + self.assertEqual(zone['transfers_success'], totalXFRCount) + self.assertEqual(zone['transfers_failed'], failedXFRCount) + +badrpzServerPort = 4251 +badrpzServer = BadRPZServer(badrpzServerPort) + +class RPZXFRIncompleteRecursorTest(RPZIncompleteRecursorTest): + """ + This test makes sure that we correctly detect incomplete RPZ zones via AXFR then IXFR + """ + + global badrpzServerPort + _lua_config_file = """ + -- The first server is a bogus one, to test that we correctly fail over to the second one + rpzMaster({'127.0.0.1:9999', '127.0.0.1:%d'}, 'zone.rpz.', { refresh=1 }) + """ % (badrpzServerPort) + _confdir = 'RPZXFRIncomplete' + _wsPort = 8042 + _wsTimeout = 2 + _wsPassword = 'secretpassword' + _apiKey = 'secretapikey' + _config_template = """ +auth-zones=example=configs/%s/example.zone +webserver=yes +webserver-port=%d +webserver-address=127.0.0.1 +webserver-password=%s +api-key=%s +""" % (_confdir, _wsPort, _wsPassword, _apiKey) + + @classmethod + def generateRecursorConfig(cls, confdir): + authzonepath = os.path.join(confdir, 'example.zone') + with open(authzonepath, 'w') as authzone: + authzone.write("""$ORIGIN example. +@ 3600 IN SOA {soa} +a 3600 IN A 192.0.2.42 +b 3600 IN A 192.0.2.42 +c 3600 IN A 192.0.2.42 +d 3600 IN A 192.0.2.42 +e 3600 IN A 192.0.2.42 +""".format(soa=cls._SOA)) + super(RPZIncompleteRecursorTest, cls).generateRecursorConfig(confdir) + + def waitUntilCorrectSerialIsLoaded(self, serial, timeout=5): + global badrpzServer + + badrpzServer.moveToSerial(serial) + + attempts = 0 + while attempts < timeout: + currentSerial = badrpzServer.getCurrentSerial() + if currentSerial > serial: + raise AssertionError("Expected serial %d, got %d" % (serial, currentSerial)) + if currentSerial == serial: + return + + attempts = attempts + 1 + time.sleep(1) + + raise AssertionError("Waited %d seconds for the serial to be updated to %d but the serial is still %d" % (timeout, serial, currentSerial)) + + def testRPZ(self): + # First zone + self.waitUntilCorrectSerialIsLoaded(1) + self.checkRPZStats(1, 1, 1, 1, 3) # failure count includes a port 9999 attempt + + # second zone, should fail, incomplete IXFR + self.waitUntilCorrectSerialIsLoaded(2) + self.checkRPZStats(1, 1, 1, 1, 5) + + # third zone, should fail, incomplete AXFR + self.waitUntilCorrectSerialIsLoaded(3) + self.checkRPZStats(1, 1, 1, 1, 7) + diff --git a/regression-tests.recursor/config.sh b/regression-tests.recursor/config.sh index 0e7404099e2d..480f1dd9ffa8 100755 --- a/regression-tests.recursor/config.sh +++ b/regression-tests.recursor/config.sh @@ -132,7 +132,7 @@ cat > $PREFIX.12/arthur.example.net.zone < recursor.log 2>&1 & sleep 3 if [ ! -e pdns_recursor.pid ]; then - echo Recursor did not start or did not write pdns_recursor.pid, exiting - exit 1 + cat recursor.log + echo Recursor did not start or did not write pdns_recursor.pid, exiting + exit 1 fi @@ -83,11 +84,11 @@ ANANSWER=$[(100*(${DBT_QUEUED}-${DBT_ERRORS}-${DBT_TIMEOUTS}) )/${DBT_QUEUED}] if [ "$ANANSWER" -ge $THRESHOLD ] then - echo recursor-bulktest >> passed_tests - RETVAL=0 + echo recursor-bulktest >> passed_tests + RETVAL=0 else - echo recursor-bulktest >> failed_tests - RETVAL=1 + echo recursor-bulktest >> failed_tests + RETVAL=1 fi echo "$DBT_OKPERCENTAGE% of domains resolved" > recursor-bulktest/diff diff --git a/regression-tests/tests/1dyndns-update-add-delete-mx/expected_result b/regression-tests/tests/1dyndns-update-add-delete-mx/expected_result index cc504a97949b..c25d426d076b 100644 --- a/regression-tests/tests/1dyndns-update-add-delete-mx/expected_result +++ b/regression-tests/tests/1dyndns-update-add-delete-mx/expected_result @@ -10,9 +10,9 @@ Answer: ;; ZONE SECTION: ;test.dyndns. IN SOA +0 test.dyndns. IN MX 3000 10 host-1.test.dyndns. +0 test.dyndns. IN MX 3000 20 host-2.test.dyndns. 0 test.dyndns. IN MX 3000 30 host-3.test.dyndns. -0 test.dyndns. IN MX 3600 10 host-1.test.dyndns. -0 test.dyndns. IN MX 3600 20 host-2.test.dyndns. 2 host-1.test.dyndns. IN A 3600 127.0.0.101 2 host-2.test.dyndns. IN A 3600 127.0.0.102 2 host-3.test.dyndns. IN A 3600 127.0.0.103 @@ -24,8 +24,8 @@ Answer: ;; ZONE SECTION: ;test.dyndns. IN SOA -0 test.dyndns. IN MX 3600 10 host-1.test.dyndns. -0 test.dyndns. IN MX 3600 20 host-2.test.dyndns. +0 test.dyndns. IN MX 3000 10 host-1.test.dyndns. +0 test.dyndns. IN MX 3000 20 host-2.test.dyndns. 2 host-1.test.dyndns. IN A 3600 127.0.0.101 2 host-2.test.dyndns. IN A 3600 127.0.0.102 Rcode: 0 (No Error), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 diff --git a/regression-tests/tests/1dyndns-update-delete-mx-prio/expected_result b/regression-tests/tests/1dyndns-update-delete-mx-prio/expected_result index 07509ebed44e..d31c55603b0c 100644 --- a/regression-tests/tests/1dyndns-update-delete-mx-prio/expected_result +++ b/regression-tests/tests/1dyndns-update-delete-mx-prio/expected_result @@ -1,5 +1,5 @@ -0 test.dyndns. IN MX 3600 10 host-1.test.dyndns. -0 test.dyndns. IN MX 3600 20 host-2.test.dyndns. +0 test.dyndns. IN MX 3000 10 host-1.test.dyndns. +0 test.dyndns. IN MX 3000 20 host-2.test.dyndns. 2 host-1.test.dyndns. IN A 3600 127.0.0.101 2 host-2.test.dyndns. IN A 3600 127.0.0.102 Rcode: 0 (No Error), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 diff --git a/regression-tests/tests/1dyndns-update-replace-mx/expected_result b/regression-tests/tests/1dyndns-update-replace-mx/expected_result index 93bef4c580b3..e14d411ef6d5 100644 --- a/regression-tests/tests/1dyndns-update-replace-mx/expected_result +++ b/regression-tests/tests/1dyndns-update-replace-mx/expected_result @@ -11,7 +11,7 @@ Answer: ;test.dyndns. IN SOA 0 test.dyndns. IN MX 3000 10 host-1.test.dyndns. -0 test.dyndns. IN MX 3600 20 host-2.test.dyndns. +0 test.dyndns. IN MX 3000 20 host-2.test.dyndns. 2 host-1.test.dyndns. IN A 3600 127.0.0.101 2 host-2.test.dyndns. IN A 3600 127.0.0.102 Rcode: 0 (No Error), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 diff --git a/regression-tests/tests/1dyndns-update-srv/expected_result b/regression-tests/tests/1dyndns-update-srv/expected_result index 0738f8dbf4da..f2032dd7fb71 100644 --- a/regression-tests/tests/1dyndns-update-srv/expected_result +++ b/regression-tests/tests/1dyndns-update-srv/expected_result @@ -16,8 +16,8 @@ Answer: ;; ZONE SECTION: ;test.dyndns. IN SOA -0 srv.test.dyndns. IN SRV 3600 1 100 389 server2. 0 srv.test.dyndns. IN SRV 3601 0 100 389 server1. +0 srv.test.dyndns. IN SRV 3601 1 100 389 server2. Rcode: 0 (No Error), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 Reply to question for qname='srv.test.dyndns.', qtype=SRV 1 test.dyndns. IN SOA 3600 ns1.test.dyndns. ahu.example.dyndns. [serial] 28800 7200 604800 86400 diff --git a/regression-tests/tests/dname-too-long-synth/command b/regression-tests/tests/dname-too-long-synth/command new file mode 100755 index 000000000000..b6c9f647fde1 --- /dev/null +++ b/regression-tests/tests/dname-too-long-synth/command @@ -0,0 +1,2 @@ +#!/bin/sh +cleandig 1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.123456.www.d.test.com A dnssec diff --git a/regression-tests/tests/dname-too-long-synth/description b/regression-tests/tests/dname-too-long-synth/description new file mode 100644 index 000000000000..7b229f21db64 --- /dev/null +++ b/regression-tests/tests/dname-too-long-synth/description @@ -0,0 +1,2 @@ +Check that we send YXDOMAIN when we can't synthesise a too long CNAME +from a DNAME. diff --git a/regression-tests/tests/dname-too-long-synth/expected_result b/regression-tests/tests/dname-too-long-synth/expected_result new file mode 100644 index 000000000000..dd2d6579ff96 --- /dev/null +++ b/regression-tests/tests/dname-too-long-synth/expected_result @@ -0,0 +1,4 @@ +0 d.test.com. IN DNAME 3600 d2.test2.com. +2 . IN OPT 32768 +Rcode: 6 (Name Exists when it should not), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 +Reply to question for qname='1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.123456.www.d.test.com.', qtype=A diff --git a/regression-tests/tests/dname-too-long-synth/expected_result.dnssec b/regression-tests/tests/dname-too-long-synth/expected_result.dnssec new file mode 100644 index 000000000000..c3054a93f919 --- /dev/null +++ b/regression-tests/tests/dname-too-long-synth/expected_result.dnssec @@ -0,0 +1,5 @@ +0 d.test.com. IN DNAME 3600 d2.test2.com. +0 d.test.com. IN RRSIG 3600 DNAME 13 3 3600 [expiry] [inception] [keytag] test.com. ... +2 . IN OPT 32768 +Rcode: 6 (Name Exists when it should not), RD: 0, QR: 1, TC: 0, AA: 1, opcode: 0 +Reply to question for qname='1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.1234567890.123456.www.d.test.com.', qtype=A diff --git a/tasks.py b/tasks.py index 6b3ec97a7489..74f14b5dcb65 100644 --- a/tasks.py +++ b/tasks.py @@ -1,10 +1,12 @@ from invoke import task from invoke.exceptions import Failure, UnexpectedExit +import os import sys import time all_build_deps = [ + 'ccache', 'libboost-all-dev', 'libluajit-5.1-dev', 'libsodium-dev', @@ -49,6 +51,20 @@ 'libfstrm-dev', 'libsnmp-dev', ] +rec_bulk_deps = [ + 'curl', + 'libboost-all-dev', + 'libcap2', + 'libfstrm0', + 'libluajit-5.1-2', + 'libsnmp35', + 'libsodium23', + 'libssl1.1', + 'libsystemd0', + 'moreutils', + 'pdns-tools', + 'unzip' +] dnsdist_build_deps = [ 'libcap-dev', 'libcdb-dev', @@ -57,6 +73,7 @@ 'libgnutls28-dev', 'libh2o-evloop-dev', 'liblmdb-dev', + 'libnghttp2-dev', 'libre2-dev', 'libsnmp-dev', ] @@ -67,6 +84,7 @@ 'curl', 'default-jre-headless', 'dnsutils', + 'faketime', 'gawk', 'ldnsutils', 'libboost-serialization1.71.0', @@ -87,7 +105,6 @@ 'libsystemd0', 'libyaml-cpp0.6', 'libzmq3-dev', - 'pdns-recursor', 'ruby-bundler', 'ruby-dev', 'socat', @@ -105,14 +122,14 @@ def apt_fresh(c): @task def install_clang(c): """ - install clang-11 and llvm-11 + install clang-12 and llvm-12 """ - c.sudo('apt-get -qq -y --no-install-recommends install clang-11 llvm-11') + c.sudo('apt-get -qq -y --no-install-recommends install clang-12 llvm-12') @task def install_clang_runtime(c): # this gives us the symbolizer, for symbols in asan/ubsan traces - c.sudo('apt-get -qq -y --no-install-recommends install clang-11') + c.sudo('apt-get -qq -y --no-install-recommends install clang-12') @task def install_auth_build_deps(c): @@ -127,7 +144,12 @@ def setup_authbind(c): gmysql=['default-libmysqlclient-dev'], gpgsql=['libpq-dev'], lmdb=[], - remote=[] + remote=[], + bind=[], + geoip=[], + lua2=[], + tinydns=[], + authpy=[] ) @task(help={'backend': 'Backend to install test deps for, e.g. gsqlite3; can be repeated'}, iterable=['backend'], optional=['backend']) @@ -144,25 +166,32 @@ def install_auth_test_deps(c, backend): # FIXME: rename this, we do way more tha # rm jdnssec-tools-0.14.tar.gz # fi # echo 'export PATH=$HOME/jdnssec-tools-0.14/bin:$PATH' >> $BASH_ENV''') # FIXME: why did this fail with no error? - c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing') # FIXME: can this go? - # FIXME we need to start a background recursor here for some tests + c.run('touch regression-tests/tests/verify-dnssec-zone/allow-missing regression-tests.nobackend/rectify-axfr/allow-missing') # FIXME: can this go? + # FIXME we may want to start a background recursor here to make ALIAS tests more robust setup_authbind(c) +@task +def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get + c.sudo('apt-get --no-install-recommends -qq -y install ' + ' '.join(rec_bulk_deps)) + c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') + @task def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get - c.sudo('apt-get --no-install-recommends install -qq -y authbind python3-venv python3-dev default-libmysqlclient-dev libpq-dev pdns-tools libluajit-5.1-2 \ - libboost-all-dev \ - libcap2 \ - libssl1.1 \ - libsystemd0 \ - libsodium23 \ - libfstrm0 \ - libsnmp35') + c.sudo('apt-get --no-install-recommends install -qq -y ' + ' '.join(rec_bulk_deps) + ' \ + pdns-server pdns-backend-bind daemontools \ + jq libfaketime lua-posix lua-socket bc authbind \ + python3-venv python3-dev default-libmysqlclient-dev libpq-dev \ + protobuf-compiler snmpd prometheus') c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') setup_authbind(c) + c.run('sed "s/agentxperms 0700 0755 recursor/agentxperms 0777 0755/g" regression-tests.recursor-dnssec/snmpd.conf | sudo tee /etc/snmp/snmpd.conf') + c.sudo('systemctl restart snmpd') + time.sleep(5) + c.sudo('chmod 755 /var/agentx') + @task def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt-get c.sudo('apt-get install -qq -y \ @@ -175,6 +204,7 @@ def install_dnsdist_test_deps(c): # FIXME: rename this, we do way more than apt- libgnutls30 \ libh2o-evloop0.13 \ liblmdb0 \ + libnghttp2-14 \ libre2-5 \ libssl-dev \ libsystemd0 \ @@ -204,10 +234,10 @@ def ci_auth_configure(c): res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ ./configure \ - CC='clang-11' \ - CXX='clang++-11' \ + CC='clang-12' \ + CXX='clang++-12' \ --enable-option-checking=fatal \ - --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe random remote tinydns' \ + --with-modules='bind geoip gmysql godbc gpgsql gsqlite3 ldap lmdb lua2 pipe remote tinydns' \ --enable-systemd \ --enable-tools \ --enable-unit-tests \ @@ -226,11 +256,12 @@ def ci_auth_configure(c): raise UnexpectedExit(res) @task def ci_rec_configure(c): + sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) res = c.run(''' CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ ./configure \ - CC='clang-11' \ - CXX='clang++-11' \ + CC='clang-12' \ + CXX='clang++-12' \ --enable-option-checking=fatal \ --enable-unit-tests \ --enable-nod \ @@ -240,20 +271,19 @@ def ci_rec_configure(c): --with-lua=luajit \ --with-libcap \ --with-net-snmp \ - --enable-dns-over-tls \ - --enable-asan \ - --enable-ubsan''', warn=True) + --enable-dns-over-tls ''' + sanitizers, warn=True) if res.exited != 0: c.run('cat config.log') raise UnexpectedExit(res) @task def ci_dnsdist_configure(c): + sanitizers = ' '.join('--enable-'+x for x in os.getenv('SANITIZERS').split('+')) res = c.run('''CFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int" \ CXXFLAGS="-O1 -Werror=vla -Werror=shadow -Wformat=2 -Werror=format-security -Werror=string-plus-int -Wp,-D_GLIBCXX_ASSERTIONS" \ ./configure \ - CC='clang-11' \ - CXX='clang++-11' \ + CC='clang-12' \ + CXX='clang++-12' \ --enable-option-checking=fatal \ --enable-unit-tests \ --enable-dnstap \ @@ -266,9 +296,8 @@ def ci_dnsdist_configure(c): --with-libsodium \ --with-lua=luajit \ --with-libcap \ - --with-re2 \ - --enable-asan \ - --enable-ubsan''', warn=True) + --with-nghttp2 \ + --with-re2 ''' + sanitizers, warn=True) if res.exited != 0: c.run('cat config.log') raise UnexpectedExit(res) @@ -295,7 +324,8 @@ def ci_auth_install_remotebackend_ruby_deps(c): def ci_auth_run_unit_tests(c): res = c.run('make check', warn=True) if res.exited != 0: - c.run('cat pdns/test-suite.log') + c.run('cat pdns/test-suite.log', warn=True) + c.run('cat modules/remotebackend/test-suite.log', warn=True) raise UnexpectedExit(res) @task @@ -345,7 +375,46 @@ def test_api(c, product, backend=''): raise Failure('unknown product') backend_regress_tests = dict( - remote = ['pipe', 'unix', 'http', 'zeromq', 'pipe-dnssec', 'unix-dnssec', 'http-dnssec', 'zeromq-dnssec'] + bind = [ + 'bind-both', + 'bind-dnssec-both', + 'bind-dnssec-nsec3-both', + 'bind-dnssec-nsec3-optout-both', + 'bind-dnssec-nsec3-narrow', + # FIXME 'bind-dnssec-pkcs11' + ], + geoip = [ + 'geoip', + 'geoip-nsec3-narrow' + # FIXME: also run this with the mmdb we ship + ], + lua2 = [ + 'lua2', + 'lua2-dnssec' + ], + tinydns = [ + 'tinydns' + ], + remote = [ + 'remotebackend-pipe', + 'remotebackend-unix', + 'remotebackend-http', + 'remotebackend-zeromq', + 'remotebackend-pipe-dnssec', + 'remotebackend-unix-dnssec', + 'remotebackend-http-dnssec', + 'remotebackend-zeromq-dnssec' + ], + lmdb = [ + 'lmdb-nodnssec-both', + 'lmdb-both', + 'lmdb-nsec3-both', + 'lmdb-nsec3-optout-both', + 'lmdb-nsec3-narrow' + ], + gmysql = ['gmysql', 'gmysql-nodnssec-both', 'gmysql-nsec3-both', 'gmysql-nsec3-optout-both', 'gmysql-nsec3-narrow', 'gmysql_sp-both'], + gpgsql = ['gpgsql', 'gpgsql-nodnssec-both', 'gpgsql-nsec3-both', 'gpgsql-nsec3-optout-both', 'gpgsql-nsec3-narrow', 'gpgsql_sp-both'], + gsqlite3 = ['gsqlite3', 'gsqlite3-nodnssec-both', 'gsqlite3-nsec3-both', 'gsqlite3-nsec3-optout-both', 'gsqlite3-nsec3-narrow'], ) @task @@ -353,11 +422,28 @@ def test_auth_backend(c, backend): if backend == 'remote': ci_auth_install_remotebackend_ruby_deps(c) + if backend == 'authpy': + with c.cd('regression-tests.auth-py'): + c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests') + return + with c.cd('regression-tests'): - for t in backend_regress_tests[backend]: + if backend == 'lua2': + c.run('touch trustedkeys') # avoid silly error during cleanup + for variant in backend_regress_tests[backend]: # FIXME this long line is terrible - # FIXME this appends 'backend' but that's only correct for 'remote' - c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {backend}backend-{t}') + c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./start-test-stop 5300 {variant}') + + if backend == 'gsqlite3': + with c.cd('regression-tests.nobackend'): + c.run(f'PDNS=/opt/pdns-auth/sbin/pdns_server PDNS2=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig NOTIFY=/opt/pdns-auth/bin/pdns_notify NSEC3DIG=/opt/pdns-auth/bin/nsec3dig SAXFR=/opt/pdns-auth/bin/saxfr ZONE2SQL=/opt/pdns-auth/bin/zone2sql ZONE2LDAP=/opt/pdns-auth/bin/zone2ldap ZONE2JSON=/opt/pdns-auth/bin/zone2json PDNSUTIL=/opt/pdns-auth/bin/pdnsutil PDNSCONTROL=/opt/pdns-auth/bin/pdns_control PDNSSERVER=/opt/pdns-auth/sbin/pdns_server SDIG=/opt/pdns-auth/bin/sdig GMYSQLHOST=127.0.0.1 GMYSQL2HOST=127.0.0.1 MYSQL_HOST="127.0.0.1" PGHOST="127.0.0.1" PGPORT="5432" ./runtests') + c.run('/opt/pdns-auth/bin/pdnsutil test-algorithms') + return + +@task +def test_ixfrdist(c): + with c.cd('regression-tests.ixfrdist'): + c.run('IXFRDISTBIN=/opt/pdns-auth/bin/ixfrdist ./runtests') @task def test_dnsdist(c): @@ -366,3 +452,25 @@ def test_dnsdist(c): c.run('ls -al /var/agentx/master') with c.cd('regression-tests.dnsdist'): c.run('DNSDISTBIN=/opt/dnsdist/bin/dnsdist ./runtests') + +@task +def test_regression_recursor(c): + c.run('/opt/pdns-recursor/sbin/pdns_recursor --version') + c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y LIBFAKETIME=/bin/false ./build-scripts/test-recursor test_RecDnstap.py') + c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control SKIP_IPV6_TESTS=y ./build-scripts/test-recursor -I test_RecDnstap.py') + +@task +def test_bulk_recursor(c, threads, mthreads, shards): + # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load + with c.cd('regression-tests'): + c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip') + c.run('unzip top-1m.csv.zip -d .') + c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*') + c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./timestamp ./recursor-test 5300 100 {threads} {mthreads} {shards}') + +# this is run always +def setup(): + if '/usr/lib/ccache' not in os.environ['PATH']: + os.environ['PATH']='/usr/lib/ccache:'+os.environ['PATH'] + +setup()