CI: add support for getting ccache cache from S3
Add support for getting ccache cache from S3. ccache is archieved in a tar and downloaded from S3 Cloud Storage. For push events, ccache is then uplodaed back to S3 to refresh and have a ccache cache always fresh. An additional workflow is added to upload files to an S3 Cloud Storage from artifacts uplodaed to github. The minio tool is used to upload files to S3. If the ccache can't be downloaded from s3, we fallback to github cache system. Also limit s3 upload to the openwrt repository since external fork won't have (obviously) the required secrtes to upload data to the S3 Cloud Storage. Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>
This commit is contained in:
Christian Marangi
parent
ff66a7c1c0
commit
ebbc806d30
40
.github/workflows/build.yml
vendored
40
.github/workflows/build.yml
vendored
|
|
@ -60,6 +60,8 @@ on:
|
|||
ccache_type:
|
||||
type: string
|
||||
default: kernel
|
||||
upload_ccache_cache:
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -232,6 +234,20 @@ jobs:
|
|||
echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
|
||||
echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Download and extract ccache cache from s3
|
||||
id: restore-ccache-cache-s3
|
||||
if: inputs.use_ccache_cache == true
|
||||
working-directory: openwrt
|
||||
run: |
|
||||
ENDPOINT=https://storage.googleapis.com
|
||||
BUCKET=openwrt-ci-cache
|
||||
CCACHE_TAR=ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar
|
||||
|
||||
if curl -o /dev/null -s --head --fail $ENDPOINT/$BUCKET/$CCACHE_TAR; then
|
||||
wget -O - $ENDPOINT/$BUCKET/$CCACHE_TAR | tar -xf -
|
||||
echo "cache-hit=true" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Fix permission
|
||||
run: |
|
||||
chown -R buildbot:buildbot openwrt
|
||||
|
|
@ -256,7 +272,7 @@ jobs:
|
|||
|
||||
- name: Restore ccache cache
|
||||
id: restore-ccache-cache
|
||||
if: inputs.use_ccache_cache == true
|
||||
if: inputs.use_ccache_cache == true && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/restore@v3
|
||||
with:
|
||||
path: openwrt/.ccache
|
||||
|
|
@ -498,7 +514,8 @@ jobs:
|
|||
path: "openwrt/logs"
|
||||
|
||||
- name: Delete already present ccache cache
|
||||
if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true && github.event_name == 'push'
|
||||
if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true &&
|
||||
github.event_name == 'push' && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
|
||||
uses: octokit/request-action@v2.x
|
||||
with:
|
||||
route: DELETE /repos/{repository}/actions/caches?key={key}
|
||||
|
|
@ -508,12 +525,29 @@ jobs:
|
|||
INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
|
||||
|
||||
- name: Save ccache cache
|
||||
if: inputs.use_ccache_cache == true && github.event_name == 'push'
|
||||
if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
|
||||
steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v3
|
||||
with:
|
||||
path: openwrt/.ccache
|
||||
key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
|
||||
|
||||
- name: Archive ccache
|
||||
if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
|
||||
inputs.upload_ccache_cache == true
|
||||
shell: su buildbot -c "sh -e {0}"
|
||||
working-directory: openwrt
|
||||
run: tar -cf ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar .ccache
|
||||
|
||||
- name: Upload ccache cache
|
||||
if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
|
||||
inputs.upload_ccache_cache == true
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ inputs.target }}-${{ inputs.subtarget }}-ccache-cache
|
||||
path: openwrt/ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar
|
||||
retention-days: 1
|
||||
|
||||
- name: Find external toolchain name
|
||||
id: get-toolchain-name
|
||||
if: inputs.upload_external_toolchain == true
|
||||
|
|
|
|||
18
.github/workflows/kernel.yml
vendored
18
.github/workflows/kernel.yml
vendored
|
|
@ -117,6 +117,7 @@ jobs:
|
|||
subtarget: ${{ matrix.subtarget }}
|
||||
build_kernel: true
|
||||
build_all_kmods: true
|
||||
upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }}
|
||||
|
||||
check-kernel-patches:
|
||||
name: Check Kernel patches
|
||||
|
|
@ -133,3 +134,20 @@ jobs:
|
|||
target: ${{ matrix.target }}
|
||||
subtarget: ${{ matrix.subtarget }}
|
||||
|
||||
upload-ccache-cache-in-s3:
|
||||
if: github.event_name == 'push' && github.repository_owner == 'openwrt'
|
||||
name: Upload ccache cache to s3
|
||||
needs: [determine_targets, build]
|
||||
strategy:
|
||||
fail-fast: False
|
||||
matrix:
|
||||
include: ${{fromJson(needs.determine_targets.outputs.targets_subtargets)}}
|
||||
secrets:
|
||||
s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }}
|
||||
s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }}
|
||||
uses: ./.github/workflows/upload-file-s3.yml
|
||||
with:
|
||||
endpoint: https://storage.googleapis.com
|
||||
bucket: openwrt-ci-cache
|
||||
download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache
|
||||
filename: ccache-kernel-${{ matrix.target }}-${{ matrix.subtarget }}.tar
|
||||
|
|
|
|||
23
.github/workflows/packages.yml
vendored
23
.github/workflows/packages.yml
vendored
|
|
@ -54,4 +54,27 @@ jobs:
|
|||
build_all_modules: true
|
||||
build_full: true
|
||||
ccache_type: packages
|
||||
upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }}
|
||||
|
||||
upload-ccache-cache-in-s3:
|
||||
if: github.event_name == 'push' && github.repository_owner == 'openwrt'
|
||||
name: Upload ccache cache to s3
|
||||
needs: build
|
||||
strategy:
|
||||
fail-fast: False
|
||||
matrix:
|
||||
include:
|
||||
- target: malta
|
||||
subtarget: be
|
||||
- target: x86
|
||||
subtarget: 64
|
||||
secrets:
|
||||
s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }}
|
||||
s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }}
|
||||
uses: ./.github/workflows/upload-file-s3.yml
|
||||
with:
|
||||
endpoint: https://storage.googleapis.com
|
||||
bucket: openwrt-ci-cache
|
||||
download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache
|
||||
filename: ccache-packages-${{ matrix.target }}-${{ matrix.subtarget }}.tar
|
||||
|
||||
|
|
|
|||
46
.github/workflows/upload-file-s3.yml
vendored
Normal file
46
.github/workflows/upload-file-s3.yml
vendored
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
name: Upload File to S3
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
secrets:
|
||||
s3_access_key:
|
||||
s3_secret_key:
|
||||
inputs:
|
||||
endpoint:
|
||||
required: true
|
||||
type: string
|
||||
bucket:
|
||||
required: true
|
||||
type: string
|
||||
download_id:
|
||||
required: true
|
||||
type: string
|
||||
filename:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
upload-file-in-s3:
|
||||
name: Upload file in S3
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Install minio
|
||||
run: |
|
||||
curl https://dl.min.io/client/mc/release/linux-amd64/mc \
|
||||
--create-dirs \
|
||||
-o $GITHUB_WORKSPACE/minio-binaries/mc
|
||||
|
||||
chmod +x $GITHUB_WORKSPACE/minio-binaries/mc
|
||||
echo $GITHUB_WORKSPACE/minio-binaries/ >> $GITHUB_PATH
|
||||
|
||||
- name: Setup minio
|
||||
run: mc alias set s3 ${{ inputs.endpoint }} ${{ secrets.s3_access_key }} ${{ secrets.s3_secret_key }}
|
||||
|
||||
- name: Download file
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: ${{ inputs.download_id }}
|
||||
|
||||
- name: Upload file to s3
|
||||
run: mc cp ${{ inputs.filename }} s3/${{ inputs.bucket }}/
|
||||
Loading…
Reference in New Issue
Block a user