forked from feast-dev/feast
-
Notifications
You must be signed in to change notification settings - Fork 0
252 lines (247 loc) · 10.7 KB
/
build_wheels.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
name: build_wheels
# Call this workflow from other workflows in the repository by specifying "uses: ./.github/workflows/build_wheels.yml"
# Developers who are starting a new release should use this workflow to ensure wheels will be built correctly.
# Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly.
on:
workflow_dispatch:
tags:
- 'v*.*.*'
workflow_call:
jobs:
get-version:
runs-on: ubuntu-latest
outputs:
release_version: ${{ steps.get_release_version.outputs.release_version }}
version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
steps:
- uses: actions/checkout@v2
- name: Get release version
id: get_release_version
run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/}
- name: Get release version without prefix
id: get_release_version_without_prefix
env:
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
run: |
echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1}
- name: Get highest semver
id: get_highest_semver
env:
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
run: |
source infra/scripts/setup-common-functions.sh
SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$'
if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then
echo ::set-output name=highest_semver_tag::$(get_tag_release -m)
fi
- name: Check output
env:
RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }}
VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }}
HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }}
run: |
echo $RELEASE_VERSION
echo $VERSION_WITHOUT_PREFIX
echo $HIGHEST_SEMVER_TAG
build-python-wheel:
name: Build wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-latest, macos-10.15 ]
steps:
- uses: actions/checkout@v2
- name: Setup Node
uses: actions/setup-node@v2
with:
node-version: '17.x'
registry-url: 'https://registry.npmjs.org'
- name: Build UI
run: make build-ui
- name: Build wheels
uses: pypa/[email protected]
env:
CIBW_BUILD: "cp3*_x86_64"
CIBW_SKIP: "cp36-* cp37-* *-musllinux_x86_64 cp310-macosx_x86_64"
CIBW_ARCHS: "native"
CIBW_ENVIRONMENT: >
COMPILE_GO=True PATH=$PATH:/usr/local/go/bin
CIBW_BEFORE_ALL_LINUX: |
curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz
tar -C /usr/local -xzf go.tar.gz
go version
yum -y update &&
yum install -y epel-release || yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm &&
yum install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm &&
yum install -y --enablerepo=epel arrow-devel # For C++
CIBW_BEFORE_ALL_MACOS: |
brew install apache-arrow
curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
sudo installer -pkg python.pkg -target /
# There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.
CIBW_BEFORE_BUILD: |
make install-protoc-dependencies
make install-go-proto-dependencies
make install-go-ci-dependencies
git status
git restore go.mod go.sum
git restore sdk/python/feast/ui/yarn.lock
CIBW_BEFORE_TEST: "cd {project} && git status"
# py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881.
- name: Build py310 specific wheels for macos
if: matrix.os == 'macos-10.15'
uses: pypa/[email protected]
env:
CIBW_BUILD: "cp310-macosx_x86_64"
CIBW_ARCHS: "native"
# Need this environment variable because of this issue: https://github.com/pypa/cibuildwheel/issues/952.
CIBW_ENVIRONMENT: >
_PYTHON_HOST_PLATFORM=macosx-10.15-x86_64
# There's a `git restore` in here because remnant go.mod, go.sum changes from the build mess up the wheel naming.
CIBW_BEFORE_BUILD: |
git status
git restore go.mod go.sum
git restore sdk/python/feast/ui/yarn.lock
brew install apache-arrow
- uses: actions/upload-artifact@v2
with:
name: wheels
path: ./wheelhouse/*.whl
build-source-distribution:
name: Build source distribution
runs-on: macos-10.15
steps:
- uses: actions/checkout@v2
- name: Setup Python
id: setup-python
uses: actions/setup-python@v2
with:
python-version: "3.10"
architecture: x64
- name: Setup Node
uses: actions/setup-node@v2
with:
node-version: '17.x'
registry-url: 'https://registry.npmjs.org'
- name: Build and install dependencies
# There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.
run: |
pip install -U pip setuptools wheel twine
make install-protoc-dependencies
make install-go-proto-dependencies
make install-go-ci-dependencies
make build-ui
git status
git restore go.mod go.sum
git restore sdk/python/feast/ui/yarn.lock
- name: Build
run: |
python3 setup.py sdist
- uses: actions/upload-artifact@v2
with:
name: wheels
path: dist/*
verify-python-wheels:
runs-on: ${{ matrix.os }}
needs: [build-python-wheel, build-source-distribution, get-version]
strategy:
matrix:
os: [ubuntu-latest, macos-10.15 ]
python-version: [ "3.8", "3.9", "3.10"]
from-source: [ True, False ]
env:
# this script is for testing servers
# it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself
TEST_SCRIPT: |
timeout 10s $@ & pid=$!
wait $pid
ret=$?
if [[ $ret -ne 124 ]]
then
exit $ret
else
echo "Succeeded!"
fi
VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }}
steps:
- name: Setup Python
id: setup-python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
architecture: x64
- uses: actions/setup-go@v3
with:
go-version: '>=1.17.0'
- uses: actions/download-artifact@v2
with:
name: wheels
path: dist
- name: Install wheel
if: ${{ !matrix.from-source }}
# try to install all wheels; only the current platform wheel should be actually installed
run: |
cd dist/
pip install wheel
for f in *.whl; do pip install $f || true; done
- name: Install apache-arrow on ubuntu
if: ${{ matrix.from-source && matrix.os == 'ubuntu-latest' }}
run: |
sudo apt update
sudo apt install -y -V ca-certificates lsb-release wget
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
sudo apt update
sudo apt install -y -V libarrow-dev
- name: Install apache-arrow on macos
if: ${{ matrix.from-source && matrix.os == 'macos-10.15' && matrix.python-version != '3.10' }}
run: brew install apache-arrow
- name: Install dist with go
if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}}
env:
COMPILE_GO: "True"
run: |
pip install 'grpcio-tools==1.47.0' 'pybindgen==0.22.0'
go install google.golang.org/protobuf/cmd/[email protected]
go install google.golang.org/grpc/cmd/[email protected]
pip install dist/*tar.gz
# py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881
- name: Install dist w/o go
if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}}
run: pip install dist/*tar.gz
- name: Install OS X dependencies
if: matrix.os == 'macos-10.15'
run: brew install coreutils
# Validate that the feast version installed is not development and is the correct version of the tag we ran it off of.
- name: Validate Feast Version
run: |
VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+'
OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$'
VERSION_OUTPUT=$(feast version)
VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX")
OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX")
if [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then
echo "Correct Feast Version Installed"
else
echo "$VERSION_OUTPUT from installed wheel is not in the correct format or doesn't have the right version $VERSION."
exit 1
fi
- name: Smoke test
run: |
feast init test_repo
cd test_repo/feature_repo
feast apply
echo "$TEST_SCRIPT" > run-and-wait.sh
bash run-and-wait.sh feast serve
bash run-and-wait.sh feast ui
# We disable this test for the Python 3.10 binary since it does not include Go.
- name: Smoke test with go
if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest'
run: |
cd test_repo/feature_repo
feast apply
echo "$TEST_SCRIPT" > run-and-wait.sh
pip install cffi
printf "\ngo_feature_serving: True" >> feature_store.yaml
bash run-and-wait.sh feast serve