Merge pull request #1228 from pi-hole/development

Pi-hole FTL v5.11
pi-hole · Oct 23, 2021 · a296265 · a296265
2 parents 2f2b5bf + 7513bfa
commit a296265
Show file tree

Hide file tree

Showing 30 changed files with 1,004 additions and 664 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -27,18 +27,6 @@ version: 2
           ssh-keyscan -H $SSH_HOST >> ~/.ssh/known_hosts
           sftp -b - $SSH_USER@$SSH_HOST <<< "-mkdir html/${DIR}
           put ${BIN_NAME}* html/${DIR}"
-    - run:
-        name: "Verify uploaded binary"
-        command: |
-          [ -z "${CIRCLE_PR_USERNAME}" ] || exit 0
-          [ "${CIRCLE_PROJECT_USERNAME}" == "pi-hole" ] || exit 0
-          DIR="${CIRCLE_TAG:-${CIRCLE_BRANCH}}"
-          mkdir download
-          cd download
-          wget "https://ftl.pi-hole.net/${DIR}/${BIN_NAME}"
-          wget "https://ftl.pi-hole.net/${DIR}/${BIN_NAME}.sha1"
-          sha1sum -c "${BIN_NAME}.sha1"
-          cd ..
     - run:
         name: "Test"
         command: |
@@ -47,7 +35,7 @@ version: 2
 
 .docker_template: &docker_template
   docker:
-    - image: pihole/ftl-build:v1.8-$CIRCLE_JOB
+    - image: pihole/ftl-build:v1.16-$CIRCLE_JOB
   <<: *job_steps
 
 jobs:

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,4 +1,4 @@
-name: Build and test
+name: Build, Test, Deploy
 
 on:
   push:
@@ -10,65 +10,64 @@ on:
 
 jobs:
   build:
-
+    if: |
+      github.event_name == 'push'
+      || github.event_name == 'release'
+      || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository)
+    outputs:
+      GIT_TAG: ${{ steps.variables.outputs.GIT_TAG }}
+      GIT_BRANCH: ${{ steps.variables.outputs.GIT_BRANCH }}
+      OUTPUT_DIR: ${{ steps.variables.outputs.OUTPUT_DIR }}
+    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         include:
-         - arch: armv4t
-           bin_name: pihole-FTL-armv4-linux-gnueabi
-         - arch: armv5te
-           bin_name: pihole-FTL-armv5-linux-gnueabi
-         - arch: armv6hf
-           bin_name:  pihole-FTL-armv6-linux-gnueabihf
-         - arch: armv7hf
-           bin_name: pihole-FTL-armv7-linux-gnueabihf
-         - arch: armv8a
-           bin_name: pihole-FTL-armv8-linux-gnueabihf
-         - arch: aarch64
-           bin_name: pihole-FTL-aarch64-linux-gnu
-         - arch: x86_64
-           bin_name: pihole-FTL-linux-x86_64
-         - arch: x86_64-musl
-           bin_name: pihole-FTL-musl-linux-x86_64
-         - arch: x86_32
-           bin_name: pihole-FTL-linux-x86_32
-
-    container: pihole/ftl-build:v1.10-${{ matrix.arch }}
-
-    runs-on: ubuntu-latest
-    continue-on-error: true
-
+          - arch: x86_64
+            bin_name: pihole-FTL-linux-x86_64
+          - arch: x86_64-musl
+            bin_name: pihole-FTL-musl-linux-x86_64
+          - arch: x86_32
+            bin_name: pihole-FTL-linux-x86_32
+          - arch: armv4t
+            bin_name: pihole-FTL-armv4-linux-gnueabi
+          - arch: armv5te
+            bin_name: pihole-FTL-armv5-linux-gnueabi
+          - arch: armv6hf
+            bin_name:  pihole-FTL-armv6-linux-gnueabihf
+          - arch: armv7hf
+            bin_name: pihole-FTL-armv7-linux-gnueabihf
+          - arch: armv8a
+            bin_name: pihole-FTL-armv8-linux-gnueabihf
+          - arch: aarch64
+            bin_name: pihole-FTL-aarch64-linux-gnu
+    container: ghcr.io/pi-hole/ftl-build:v1.16-${{ matrix.arch }}
     steps:
       -
-        name: Update git (until we update base image)
-        if: ${{ matrix.arch != 'x86_64-musl' }}
-        run: |
-          echo "deb http://deb.debian.org/debian stretch-backports main contrib non-free" >> /etc/apt/sources.list;
-          apt-get update
-          apt-get -t stretch-backports install git -y
+        name: Checkout code
+        uses: actions/checkout@v2
       -
-        name: Get Branch/Tag Name
-        id: branch_name
+        name: Calculate required variables
+        id: variables
         run: |
-          GIT_BRANCH=${GITHUB_REF#refs/*/}
           GIT_TAG=${{ github.event.release.tag_name }}
+          # If GIT_TAG is set then GIT BRANCH should be "master", else set it from GITHUB_REF
+          GIT_BRANCH=$([ -n "${GIT_TAG}" ] && echo "master" || echo "${GITHUB_REF#refs/*/}")
           echo ::set-output name=GIT_BRANCH::${GIT_BRANCH}
           echo ::set-output name=GIT_TAG::${GIT_TAG}
           echo ::set-output name=OUTPUT_DIR::${GIT_TAG:-${GIT_BRANCH}}
-      -
-        name: Checkout code
-        uses: actions/checkout@v2
       -
         name: "Build"
         run: |
-          bash .circleci/build-CI.sh "-DSTATIC=${STATIC}" "${{ steps.branch_name.outputs.GIT_BRANCH }}" "${{ steps.branch_name.outputs.GIT_TAG }}" "${{ matrix.arch }}"
+          bash .circleci/build-CI.sh "-DSTATIC=${STATIC}" "${{ steps.variables.outputs.GIT_BRANCH }}" "${{ steps.variables.outputs.GIT_TAG }}" "${{ matrix.arch }}"
       -
         name: "Binary checks"
         run: |
           export CIRCLE_JOB="${{ matrix.arch }}"
           bash test/arch_test.sh
       -
-        name: "Tests"
+        name: "Test x86_32/64 binaries"
+        if: matrix.arch == 'x86_64' || matrix.arch == 'x86_64-musl' || matrix.arch == 'x86_32'
         run: |
             export CIRCLE_JOB="${{ matrix.arch }}"
             test/run.sh
@@ -78,21 +77,40 @@ jobs:
           mv pihole-FTL "${{ matrix.bin_name }}"
           sha1sum pihole-FTL-* > ${{ matrix.bin_name }}.sha1
       -
-        name: Transfer Builds to Pi-hole server for pihole checkout
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: appleboy/scp-action@master
+        name: Upload artifacts to job for later processing
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@v2
+        with:
+          name: tmp-binary-storage
+          path: '${{ matrix.bin_name }}*'
+
+  deploy:
+    if: github.event_name != 'pull_request'
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Binaries built in previous jobs
+        uses: actions/download-artifact@v2
+        with:
+          name: tmp-binary-storage
+      -
+        name: Display structure of downloaded files
+        run: ls -R
+      -
+        name: Install SSH Key
+        uses: shimataro/ssh-key-action@v2
         with:
-          host: ${{ secrets.SSH_HOST }}
-          username: ${{ secrets.SSH_USER }}
-          port: ${{ secrets.SSH_PORT }}
           key: ${{ secrets.SSH_KEY }}
-          source: "${{ matrix.bin_name }}*"
-          target: "${{ steps.branch_name.outputs.OUTPUT_DIR }}"
+          known_hosts: ${{ secrets.KNOWN_HOSTS }}
+      -
+        name: Transfer Builds to Pi-hole server for pihole checkout
+        run: |
+          sftp -b - ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} <<< "-mkdir ${{ needs.build.outputs.OUTPUT_DIR }}
+          put * ${{ needs.build.outputs.OUTPUT_DIR }}"
       -
         name: Attach binaries to release
-        if: ${{ github.event_name == 'release' }}
-        uses: skx/github-action-publish-binaries@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        if: github.event_name == 'release'
+        uses: softprops/action-gh-release@v1
         with:
-          args: '${{ matrix.bin_name }}*'
+          files: |
+            *
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -90,10 +90,9 @@ separate_arguments(EXTRAWARN)
 if(STATIC STREQUAL "true")
     SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
     SET(BUILD_SHARED_LIBS OFF)
-else()
-    # -pie -fPIE: (Dynamic) position independent executable
-    set(HARDENING_FLAGS "${HARDENING_FLAGS} -pie -fPIE")
 endif()
+# -pie -fPIE: (Dynamic) position independent executable
+set(HARDENING_FLAGS "${HARDENING_FLAGS} -pie -fPIE")
 
 # -FILE_OFFSET_BITS=64: used by stat(). Avoids problems with files > 2 GB on 32bit machines
 # We define HAVE_POLL_H as this is needed for the musl builds to succeed
@@ -182,7 +181,7 @@ add_executable(pihole-FTL
 if(STATIC STREQUAL "true")
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON)
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_END_STATIC ON)
-    target_link_libraries(pihole-FTL -static-libgcc -static -static-pie)
+    target_link_libraries(pihole-FTL -static-libgcc -static -pie)
 else()
     find_library(LIBMATH m)
     target_link_libraries(pihole-FTL ${LIBMATH})
@@ -192,10 +191,10 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 
 # for DNSSEC we need the nettle (+ hogweed) crypto and the gmp math libraries
-find_library(LIBHOGWEED libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX})
-find_library(LIBGMP libgmp${CMAKE_STATIC_LIBRARY_SUFFIX})
-find_library(LIBNETTLE libnettle${CMAKE_STATIC_LIBRARY_SUFFIX})
-find_library(LIBIDN libidn${CMAKE_STATIC_LIBRARY_SUFFIX})
+find_library(LIBHOGWEED NAMES libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX} hogweed)
+find_library(LIBGMP NAMES libgmp${CMAKE_STATIC_LIBRARY_SUFFIX} gmp)
+find_library(LIBNETTLE NAMES libnettle${CMAKE_STATIC_LIBRARY_SUFFIX} nettle)
+find_library(LIBIDN NAMES libidn${CMAKE_STATIC_LIBRARY_SUFFIX} idn)
 
 target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN})
 
@@ -204,9 +203,9 @@ if(LUA_DL STREQUAL "true")
     target_link_libraries(pihole-FTL ${LIBDL})
 endif()
 
-find_library(LIBREADLINE libreadline${CMAKE_STATIC_LIBRARY_SUFFIX})
-find_library(LIBHISTORY libhistory${CMAKE_STATIC_LIBRARY_SUFFIX})
-find_library(LIBTERMCAP libtermcap${CMAKE_STATIC_LIBRARY_SUFFIX})
+find_library(LIBREADLINE NAMES libreadline${CMAKE_STATIC_LIBRARY_SUFFIX} readline)
+find_library(LIBHISTORY NAMES libhistory${CMAKE_STATIC_LIBRARY_SUFFIX} history)
+find_library(LIBTERMCAP NAMES libtermcap${CMAKE_STATIC_LIBRARY_SUFFIX} termcap)
 if(LIBREADLINE AND LIBHISTORY AND LIBTERMCAP)
     message(STATUS "Building FTL with readline support: YES")
     target_compile_definitions(FTL PRIVATE LUA_USE_READLINE)

diff --git a/src/api/api.c b/src/api/api.c
@@ -1002,10 +1002,7 @@ void getAllQueries(const char *client_message, const int *sock)
 		else
 			clientIPName = getClientIPString(query);
 
-		unsigned long delay = query->response;
-		// Check if received (delay should be smaller than 30min)
-		if(delay > 1.8e7)
-			delay = 0;
+		unsigned long delay = query->flags.response_calculated ? query->response : 0UL;
 
 		// Get domain blocked during deep CNAME inspection, if applicable
 		const char *CNAME_domain = "N/A";
@@ -1044,6 +1041,21 @@ void getAllQueries(const char *client_message, const int *sock)
 			}
 		}
 
+		// Get reply type
+		// If this is a partially cached CNAME (parts needed to be
+		// forwarded) but we never receive replies, we have to set the
+		// reply back to unknown instead of handing out "CNAME"
+		// See https://discourse.pi-hole.net/t/garbage-response-times-for-many-almost-half-at-times-cname-answers/50291/17
+		enum reply_type reply = query->flags.response_calculated ? query->reply : REPLY_UNKNOWN;
+
+		// Overwrite reply and reply time if they don't make sense for this query
+		// See same Discourse discussion as immediately above
+		if(query->status == QUERY_RETRIED || query->status == QUERY_IN_PROGRESS)
+		{
+			reply = REPLY_UNKNOWN;
+			delay = 0UL;
+		}
+
 		if(istelnet[*sock])
 		{
 			ssend(*sock,"%lli %s %s %s %i %i %i %lu %s %i %s#%u \"%s\"",
@@ -1053,7 +1065,7 @@ void getAllQueries(const char *client_message, const int *sock)
 				clientIPName,
 				query->status,
 				query->dnssec,
-				query->reply,
+				reply,
 				delay,
 				CNAME_domain,
 				regex_idx,