Skip to content

Commit

Permalink
Automate release process (#29)
Browse files Browse the repository at this point in the history
fix #8
fix #27

* Add some scripts to release
  * dev/release/prepare.sh: For a new release.
  * dev/release/tag.sh: For tagging.
  * See
    https://datafusion-contrib.github.io/datafusion-c/main/developer/release.html
    how to release a new version.

* Add support for GitHub Releases
  * Create a release page automatically when we tag a new version

* Add support for publishing .deb/.rpm to
  https://apache.jfrog.io/artifactory/arrow/
  * See also:
    https://lists.apache.org/thread/f47nmpmmydmr9613jty21sgbv83pr9tq
  • Loading branch information
kou committed Aug 22, 2022
1 parent eeb2644 commit c0ed81e
Show file tree
Hide file tree
Showing 21 changed files with 611 additions and 51 deletions.
19 changes: 5 additions & 14 deletions .github/workflows/doc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,7 @@ jobs:
libgirepository1.0-dev \
ninja-build \
valac
pip install \
Jinja2 \
Pygments \
Sphinx \
breathe \
gi-docgen \
meson \
pydata_sphinx_theme
pip install -r requirements.txt
case ${GITHUB_REF} in
refs/tags/*)
version=${GITHUB_REF#refs/tags/}
Expand Down Expand Up @@ -96,17 +89,15 @@ jobs:
path: html/
- uses: actions/checkout@v3
if: |
github.event_name == 'push' &&
(startsWith(github.ref, 'refs/tags/') ||
github.ref == 'refs/heads/main')
startsWith(github.ref, 'refs/tags/') ||
github.ref == 'refs/heads/main'
with:
ref: gh-pages
path: gh-pages
- name: Deploy
if: |
github.event_name == 'push' &&
(startsWith(github.ref, 'refs/tags/') ||
github.ref == 'refs/heads/main')
startsWith(github.ref, 'refs/tags/') ||
github.ref == 'refs/heads/main'
run: |
rm -rf gh-pages/${VERSION}
cp -a build/doc/html/ gh-pages/${VERSION}
Expand Down
60 changes: 51 additions & 9 deletions .github/workflows/package.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,47 @@ concurrency:
cancel-in-progress: true

jobs:
source:
name: Source
runs-on: ubuntu-latest
steps:
- name: Prepare
run: |
case ${GITHUB_REF} in
refs/tags/*)
version=${GITHUB_REF#refs/tags/}
;;
*)
version=${GITHUB_SHA}
;;
esac
echo "VERSION=${version}" >> ${GITHUB_ENV}
- uses: actions/checkout@v3
with:
submodules: true
path: datafusion-c-${{ env.VERSION }}
- name: Archive
run: |
rm -rf datafusion-c-${VERSION}/.git
tar cvzf datafusion-c-${VERSION}.tar.gz datafusion-c-${VERSION}
zip -r datafusion-c-${VERSION}.zip datafusion-c-${VERSION}
- uses: actions/upload-artifact@v3
with:
name: tar.gz
path: datafusion-c-${{ env.VERSION }}.tar.gz
- uses: actions/upload-artifact@v3
with:
name: zip
path: datafusion-c-${{ env.VERSION }}.zip
- uses: softprops/action-gh-release@v1
if: |
startsWith(github.ref, 'refs/tags/')
with:
body_path: datafusion-c-${{ env.VERSION }}/doc/source/news/${{ env.VERSION }}.md
files: |
datafusion-c-${{ env.VERSION }}.tar.gz
datafusion-c-${{ env.VERSION }}.zip
build:
name: Build
runs-on: ubuntu-22.04
Expand Down Expand Up @@ -84,31 +125,29 @@ jobs:
# target: debian-bookworm-arm64
# test-docker-image: arm64v8/debian:bookworm
- label: Ubuntu Focal amd64
id: debian-focal-amd64
id: ubuntu-focal-amd64
task-namespace: apt
target: ubuntu-focal
test-docker-image: ubuntu:focal
# - label: Ubuntu Focal arm64
# id: debian-focal-arm64
# id: ubuntu-focal-arm64
# task-namespace: apt
# target: ubuntu-focal-arm64
# test-docker-image: arm64v8/ubuntu:focal
- label: Ubuntu Jammy amd64
id: debian-jammy-amd64
id: ubuntu-jammy-amd64
task-namespace: apt
target: ubuntu-jammy
test-docker-image: ubuntu:jammy
# - label: Ubuntu Jammy arm64
# id: debian-jammy-arm64
# id: ubuntu-jammy-arm64
# task-namespace: apt
# target: ubuntu-jammy-arm64
# test-docker-image: arm64v8/ubuntu:jammy
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Checkout Arrow
uses: actions/checkout@v3
- name: Install dependencies
run: |
sudo apt update
Expand All @@ -118,8 +157,7 @@ jobs:
ruby
- name: Update version
if: |
!startsWith(github.ref, 'refs/tags/') &&
!startsWith(github.ref, 'refs/heads/maintenance/')
!startsWith(github.ref, 'refs/tags/')
run: |
cd package
rake version:update RELEASE_DATE=$(date +%Y-%m-%d)
Expand All @@ -145,13 +183,16 @@ jobs:
startsWith(github.ref, 'refs/tags/')
run: |
mkdir -p ${{ matrix.id }}
shopt -s globstar
cp -a package/${{ matrix.task-namespace }}/repositories/**/*.* ${{ matrix.id }}/
tar czf ${{ matrix.id }}.tar.gz ${{ matrix.id }}
echo "VERSION=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV}
- name: Upload to release
uses: softprops/action-gh-release@v1
if: |
startsWith(github.ref, 'refs/tags/')
with:
body_path: doc/source/news/${{ env.VERSION }}.md
files: |
${{ matrix.id }}.tar.gz
- name: Test
Expand All @@ -160,4 +201,5 @@ jobs:
--rm \
--volume ${PWD}:/host:ro \
${{ matrix.test-docker-image }} \
/host/package/${{ matrix.task-namespace }}/test.sh
/host/package/${{ matrix.task-namespace }}/test.sh \
local
15 changes: 15 additions & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

/vendor/
87 changes: 87 additions & 0 deletions dev/release/prepare.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/bin/bash
#
# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eux

if [ $# -lt 1 ]; then
echo "Usage: $0 version"
echo " e.g.: $0 11.0.0"
exit 1
fi

version=$1
major_version=$(echo ${version} | grep -o -E '^[0-9]+')

source_dir="$(dirname "$0")/../.."

git fetch origin --tags

branch="prepare-${version}"
git branch -D ${branch} || :
git checkout -b ${branch} origin/main

pushd "${source_dir}"
sed -i'.bak' \
-e "s/^version = \".*\"/version = \"${version}\"/" \
Cargo.toml
rm Cargo.toml.bak
git add Cargo.toml

sed -i'.bak' \
-e "s/^version = \'.*\'/version = \"${version}\"/" \
meson.build
rm meson.build.bak
git add meson.build
popd

pushd "${source_dir}/package/debian/"
sed -i'.bak' -E \
-e "s/libdatafusion[0-9]+/libdatafusion${major_version}/g" \
-e "s/libdatafusion-glib[0-9]+/libdatafusion-glib${major_version}/g" \
-e "s/gir1\.2-datafusion-[0-9]+/gir1.2-datafusion-${major_version}/g" \
control
rm control.bak
git add control
git mv gir1.2-datafusion-*.install \
gir1.2-datafusion-${major_version}.0.install
git mv libdatafusion-glib[[:alnum:]]*.install \
libdatafusion-glib${major_version}.install
git mv libdatafusion[[:alnum:]]*.install \
libdatafusion${major_version}.install
popd

pushd "${source_dir}/doc/"
latest_news_md=$(ls source/news/*.md | sort -n -r | head -n1)
sed \
-e "s/^# .*\$/# ${version} - $(date +%Y-%m-%d)/" \
${latest_news_md} > source/news/${version}.md
git add source/news/${version}.md
sed -i'.bak' \
-e "N; /^\.\. toctree::/ a \ \ \ news/${version}" \
source/news.rst
rm source/news.rst.bak
git add source/news.rst
sed -i'.bak' \
-e "\,^ 'source' / 'news\.rst', i \ \ 'source' / 'news' / '${version}.md'," \
meson.build
rm meson.build.bak
git add meson.build
popd

pushd "${source_dir}/package"
rake version:update
git add .
popd
25 changes: 25 additions & 0 deletions dev/release/tag.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/bin/bash
#
# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eux

source_dir="$(dirname "$0")/../.."

version=$(grep "^version" "${source_dir}/Cargo.toml" | \
head -n1 | \
grep -o "[0-9.]*")
git tag -a -m "Release ${version}!!!" ${version}
git push origin ${version}
63 changes: 63 additions & 0 deletions dev/release/verify-packages.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/bin/bash
#
# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eux

if [ $# -lt 1 ]; then
echo "Usage: $0 rc"
echo " $0 staging-rc"
echo " $0 release"
echo " $0 staging-release"
echo " e.g.: $0 rc # Verify RC packages"
echo " e.g.: $0 staging-rc # Verify RC packages on staging"
echo " e.g.: $0 release # Verify release packages"
echo " e.g.: $0 staging-release # Verify release packages on staging"
exit 1
fi

verify_type="$1"

source_dir="$(dirname "$0")/../.."

pushd "${source_dir}"
for dir in package/{apt,yum}/*; do
if [ ! -d "${dir}" ]; then
continue
fi
base_name=${dir##*/}
distribution=
code_name=
case ${base_name} in
*-aarch64|*-arm64)
;;
almalinux-*|debian-*|ubuntu-*)
distribution=${base_name%-*}
code_name=${base_name#*-}
;;
esac
if [ "${distribution}" = "" -o "#{code_name}" = "" ]; then
continue
fi
package_type=$(basename "$(dirname "${dir}")")
docker run \
--rm \
--volume "$PWD:/host" \
-it \
${distribution}:${code_name} \
/host/package/${package_type}/test.sh \
"${verify_type}"
done
popd
2 changes: 2 additions & 0 deletions doc/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ depend_files = files(
'source' / 'index.rst',
'source' / 'install.rst',
'source' / 'introduction.rst',
'source' / 'news' / '10.0.0.md',
'source' / 'news.rst',
'source' / 'raw-c-api.rst',
)
depend_files += [gi_docgen_toml]
Expand Down
5 changes: 5 additions & 0 deletions doc/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@

extensions = [
'breathe',
'myst_parser',
]
source_suffix = {
'.rst': 'restructuredtext',
'.md': 'markdown',
}

breathe_default_project = 'datafusion-c'
breathe_domain_by_extension = {'h' : 'c'}
Expand Down
19 changes: 19 additions & 0 deletions doc/source/developer.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
.. Copyright 2022 Sutou Kouhei <kou@clear-code.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
Developer
=========

.. toctree::

developer/release
Loading

0 comments on commit c0ed81e

Please sign in to comment.