runit

 #!/bin/bash
set -ex
source ../pypidiff/token
export GIT_SSH_COMMAND="ssh -i $HOME/pypidiff/diffbot.key"

TODAY=$(date +'%Y%m%d')
YESTERDAY=$(date +'%Y%m%d' -d 'yesterday')
NOW=$(date +'%Y%m%d%H%M')
PARM="-C $TODAY"

LIST=$(mktemp)
gh repo list pypi-diff > $LIST

if ! grep -q $TODAY $LIST; then
    echo "Create new repo $TODAY"
    gh repo create pypi-diff/$TODAY --public -p  pypi-diff/template -d "Daily generated diffs for pypi package releases"
    # cloning from template takes some time
    sleep 20

    git clone ssh://git@github.com/pypi-diff/$TODAY
    git $PARM config user.name "pypidiff"
    git $PARM config user.email "fbonly@schiach.de"

    if [ -e $YESTERDAY/serial ]; then
        cp $YESTERDAY/serial $TODAY/serial
    else
        curl https://raw.githubusercontent.com/pypi-diff/$YESTERDAY/master/serial > $TODAY/serial
    fi

    # start workflows for scanning results
    for WF in $(gh workflow list -R github.com/pypi-diff/$YESTERDAY | awk '{print $1}'); do
        gh workflow run -R github.com/pypi-diff/$YESTERDAY $WF || true
    done
fi

rm -f $LIST

if [ ! -e $TODAY ]; then
    git clone ssh://git@github.com/pypi-diff/$TODAY
fi

python3 fetch.py --serial $TODAY/serial --output $TODAY --logfile $TODAY/fetcher.log --worker 10

# do not try to hit any limits ..
find $TODAY  -type f -not -path '*/\.git/*' ! -name "*.gz" ! -name "fetcher.log" -and -size +50M -exec gzip -f {} \;
git $PARM fetch
git $PARM add .
git $PARM commit -m "changes: $NOW" -a
git $PARM push