Deduplicate files before upload

Signed-off-by: Addisu Z. Taddese <[email protected]>

Author: azeey

.github/workflows/deploy.yml

@@ -81,21 +81,30 @@ jobs:
     concurrency:
       group: pages
       cancel-in-progress: false
+    env:
+      BUILD_DIR: merged_artifacts/
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
         with:
           merge-multiple: true
+          path: ${{ env.BUILD_DIR }}
+      - name: Deduplicate files
+        run: |
+          sudo apt-get install -y rdfind symlinks
+          bash tools/deduplicate_files.sh $BUILD_DIR
       - name: Upload merged
         uses: actions/upload-artifact@v4
         with:
           name: website-docs-merged
-          path: ./
+          path: ${{ env.BUILD_DIR }}
       - name: Commit
-        uses: peaceiris/actions-gh-pages@v4
         # The workflow upto this point is good for generating a preview,
         # but only commit to deploy if we are on the master branch (not a pull request).
         if: github.ref == 'refs/heads/master'
+        uses: JamesIves/github-pages-deploy-action@v4
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./
-          keep_files: true
+          folder: ${{ env.BUILD_DIR }}
+          clean-exclude: |
+            api/*

.github/workflows/nightly-upload.yml

@@ -74,6 +74,10 @@ jobs:
         pattern: api-docs-*
         merge-multiple: false
     - run: python3 tools/restructure_doxygen_artifacts.py --input ${{steps.download.outputs.download-path}} .api-out
+    - name: Deduplicate files
+      run: |
+        sudo apt-get install -y rdfind symlinks
+        bash tools/deduplicate_files.sh .api-out
     - uses: actions/upload-artifact@v4
       with:
         name: api-docs
@@ -84,9 +88,8 @@ jobs:
       # The workflow upto this point is good for generating a preview,
       # but only commit to deploy if we are on the master branch (not a pull request).
       if: github.ref == 'refs/heads/master'
-      uses: peaceiris/actions-gh-pages@v4
+      uses: JamesIves/github-pages-deploy-action@v4
       with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: ./.api-out
-        destination_dir: api
-        keep_files: true
+        folder: ./.api-out
+        target-folder: api
+        clean: false

tools/deduplicate_files.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+# Copyright (C) 2025 Open Source Robotics Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script uses the commands rdfind and symlinks to detect duplicates and
+# replace them with symlinks leaving only one copy from the list of duplicates.
+
+if [[ $# -lt 1 ]]; then
+    echo "deduplicate_files <directory>"
+    exit 1
+fi
+
+(
+  cd ${1}
+  rdfind -makeresultsfile false -makesymlinks true .
+  symlinks -cr .
+)
+