Skip to content

Commit a77a8f6

Browse files
sudohainguyensudohainguyen
authored
Merge branch 'feast-dev:master' into sudohainguyen-patch-1
2 parents 8a0d34b + 4394696 commit a77a8f6

14 files changed

Lines changed: 303 additions & 89 deletions

File tree

‎.github/workflows/build_wheels.yml‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -62,31 +62,24 @@ jobs:
6262
registry-url: 'https://registry.npmjs.org'
6363
- name: Build UI
6464
run: make build-ui
65-
- name: Install apache-arrow on ubuntu
66-
if: matrix.os == 'ubuntu-latest'
67-
run: |
68-
sudo apt update
69-
sudo apt install -y -V ca-certificates lsb-release wget
70-
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
71-
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
72-
sudo apt update
73-
sudo apt install -y -V libarrow-dev
74-
- name: Install apache-arrow on macos
75-
if: matrix.os == 'macOS-latest'
76-
run: brew install apache-arrow
7765
- name: Build wheels
7866
uses: pypa/cibuildwheel@v2.7.0
7967
env:
8068
CIBW_BUILD: "cp3*_x86_64"
81-
CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64"
69+
CIBW_SKIP: "cp36-* cp37-* *-musllinux_x86_64 cp310-macosx_x86_64"
8270
CIBW_ARCHS: "native"
8371
CIBW_ENVIRONMENT: >
8472
COMPILE_GO=True PATH=$PATH:/usr/local/go/bin
8573
CIBW_BEFORE_ALL_LINUX: |
8674
curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz
8775
tar -C /usr/local -xzf go.tar.gz
8876
go version
77+
yum -y update &&
78+
yum install -y epel-release || yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm &&
79+
yum install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm &&
80+
yum install -y --enablerepo=epel arrow-devel # For C++
8981
CIBW_BEFORE_ALL_MACOS: |
82+
brew install apache-arrow
9083
curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg
9184
sudo installer -pkg python.pkg -target /
9285
# There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum.
@@ -111,6 +104,7 @@ jobs:
111104
CIBW_BEFORE_BUILD: |
112105
git status
113106
git restore go.mod go.sum
107+
brew install apache-arrow
114108
- uses: actions/upload-artifact@v2
115109
with:
116110
name: wheels
@@ -156,7 +150,7 @@ jobs:
156150
strategy:
157151
matrix:
158152
os: [ubuntu-latest, macos-10.15 ]
159-
python-version: [ "3.7", "3.8", "3.9", "3.10"]
153+
python-version: [ "3.8", "3.9", "3.10"]
160154
from-source: [ True, False ]
161155
env:
162156
# this script is for testing servers

‎.github/workflows/nightly-ci.yml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
name: nightly-ci
2+
3+
on:
4+
schedule:
5+
- cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST.
6+
7+
# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532
8+
#concurrency:
9+
# group: pr-integration-tests-${{ github.event.pull_request.number }}
10+
# cancel-in-progress: true
11+
12+
jobs:
13+
check_date:
14+
runs-on: ubuntu-latest
15+
name: Check latest commit
16+
outputs:
17+
WAS_EDITED: ${{ steps.check_date.outputs.WAS_EDITED }}
18+
steps:
19+
- uses: actions/checkout@v2
20+
with:
21+
ref: develop
22+
- id: check_date
23+
name: Check if there were commits in the last day
24+
if: ${{ github.event_name == 'schedule' }}
25+
run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false')
26+
27+
integration-test-python:
28+
needs: [check_date]
29+
runs-on: ${{ matrix.os }}
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
python-version: [ "3.8" ]
34+
os: [ ubuntu-latest ]
35+
env:
36+
OS: ${{ matrix.os }}
37+
PYTHON: ${{ matrix.python-version }}
38+
services:
39+
redis:
40+
image: redis
41+
ports:
42+
- 6379:6379
43+
options: >-
44+
--health-cmd "redis-cli ping"
45+
--health-interval 10s
46+
--health-timeout 5s
47+
--health-retries 5
48+
steps:
49+
- uses: actions/checkout@v2
50+
with:
51+
# pull_request_target runs the workflow in the context of the base repo
52+
# as such actions/checkout needs to be explicit configured to retrieve
53+
# code from the PR.
54+
ref: refs/pull/${{ github.event.pull_request.number }}/merge
55+
submodules: recursive
56+
- name: Setup Python
57+
uses: actions/setup-python@v2
58+
id: setup-python
59+
with:
60+
python-version: ${{ os.PYTHON }}
61+
architecture: x64
62+
- name: Setup Go
63+
id: setup-go
64+
uses: actions/setup-go@v2
65+
with:
66+
go-version: 1.18.0
67+
- name: Set up gcloud SDK
68+
uses: google-github-actions/setup-gcloud@v0
69+
with:
70+
project_id: ${{ secrets.GCP_PROJECT_ID }}
71+
service_account_key: ${{ secrets.GCP_SA_KEY }}
72+
export_default_credentials: true
73+
- name: Use gcloud CLI
74+
run: gcloud info
75+
- name: Set up AWS SDK
76+
uses: aws-actions/configure-aws-credentials@v1
77+
with:
78+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
79+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
80+
aws-region: us-west-2
81+
- name: Use AWS CLI
82+
run: aws sts get-caller-identity
83+
- name: Upgrade pip version
84+
run: |
85+
pip install --upgrade "pip>=21.3.1,<22.1"
86+
- name: Get pip cache dir
87+
id: pip-cache
88+
run: |
89+
echo "::set-output name=dir::$(pip cache dir)"
90+
- name: pip cache
91+
uses: actions/cache@v2
92+
with:
93+
path: |
94+
${{ steps.pip-cache.outputs.dir }}
95+
/opt/hostedtoolcache/Python
96+
/Users/runner/hostedtoolcache/Python
97+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
98+
restore-keys: |
99+
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
100+
- name: Install pip-tools
101+
run: pip install pip-tools
102+
- name: Install apache-arrow on ubuntu
103+
if: matrix.os == 'ubuntu-latest'
104+
run: |
105+
sudo apt update
106+
sudo apt install -y -V ca-certificates lsb-release wget
107+
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
108+
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
109+
sudo apt update
110+
sudo apt install -y -V libarrow-dev
111+
- name: Install apache-arrow on macos
112+
if: matrix.os == 'macOS-latest'
113+
run: brew install apache-arrow
114+
- name: Install dependencies
115+
run: make install-python-ci-dependencies
116+
- name: Setup Redis Cluster
117+
run: |
118+
docker pull vishnunair/docker-redis-cluster:latest
119+
docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
120+
- name: Test python
121+
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
122+
env:
123+
FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
124+
FEAST_USAGE: "False"
125+
IS_TEST: "True"
126+
SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
127+
SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }}
128+
SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
129+
SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }}
130+
SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
131+
run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread
132+
- name: Upload coverage to Codecov
133+
uses: codecov/codecov-action@v1
134+
with:
135+
token: ${{ secrets.CODECOV_TOKEN }}
136+
files: ./coverage.xml
137+
flags: integrationtests
138+
env_vars: OS,PYTHON
139+
fail_ci_if_error: true
140+
verbose: true

‎.github/workflows/pr_integration_tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,4 +191,4 @@ jobs:
191191
flags: integrationtests
192192
env_vars: OS,PYTHON
193193
fail_ci_if_error: true
194-
verbose: true
194+
verbose: true

‎.github/workflows/pr_local_integration_tests.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
name: pr-local-integration-tests
2+
# This runs local tests with containerized stubs of online stores. This is the main dev workflow
3+
4+
on:
5+
pull_request_target:
6+
types:
7+
- opened
8+
- synchronize
9+
- labeled
10+
11+
jobs:
12+
integration-test-python-local:
13+
# all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
14+
if:
15+
(github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
16+
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))
17+
runs-on: ${{ matrix.os }}
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
python-version: [ "3.8" ]
22+
os: [ ubuntu-latest ]
23+
env:
24+
OS: ${{ matrix.os }}
25+
PYTHON: ${{ matrix.python-version }}
26+
steps:
27+
- uses: actions/checkout@v2
28+
with:
29+
# pull_request_target runs the workflow in the context of the base repo
30+
# as such actions/checkout needs to be explicit configured to retrieve
31+
# code from the PR.
32+
ref: refs/pull/${{ github.event.pull_request.number }}/merge
33+
submodules: recursive
34+
- name: Setup Python
35+
uses: actions/setup-python@v2
36+
id: setup-python
37+
with:
38+
python-version: ${{ matrix.python-version }}
39+
architecture: x64
40+
- name: Upgrade pip version
41+
run: |
42+
pip install --upgrade "pip>=21.3.1,<22.1"
43+
- name: Get pip cache dir
44+
id: pip-cache
45+
run: |
46+
echo "::set-output name=dir::$(pip cache dir)"
47+
- name: pip cache
48+
uses: actions/cache@v2
49+
with:
50+
path: |
51+
${{ steps.pip-cache.outputs.dir }}
52+
/opt/hostedtoolcache/Python
53+
/Users/runner/hostedtoolcache/Python
54+
key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
55+
restore-keys: |
56+
${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
57+
- name: Install pip-tools
58+
run: pip install pip-tools
59+
- name: Install apache-arrow on ubuntu
60+
if: matrix.os == 'ubuntu-latest'
61+
run: |
62+
sudo apt update
63+
sudo apt install -y -V ca-certificates lsb-release wget
64+
wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
65+
sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
66+
sudo apt update
67+
sudo apt install -y -V libarrow-dev
68+
- name: Install dependencies
69+
run: make install-python-ci-dependencies
70+
- name: Test local integration tests
71+
if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
72+
run: make test-python-integration-local

‎CONTRIBUTING.md‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -133,17 +133,19 @@ make test-python
133133
134134
### Integration Tests
135135
There are two sets of tests you can run:
136-
1. Local integration tests (for faster development)
136+
1. Local integration tests (for faster development, tests file offline store & key online stores)
137137
2. Full integration tests (requires cloud environment setups)
138138

139139
#### Local integration tests
140-
To get local integration tests running, you'll need to have Redis setup:
140+
For this approach of running tests, you'll need to have docker set up locally: [Get Docker](https://docs.docker.com/get-docker/)
141141

142-
Redis
143-
1. Install Redis: [Quickstart](https://redis.io/topics/quickstart)
144-
2. Run `redis-server`
142+
It leverages a file based offline store to test against emulated versions of Datastore, DynamoDB, and Redis, using ephemeral containers.
145143

146-
Now run `make test-python-universal-local`
144+
These tests create new temporary tables / datasets locally only, and they are cleaned up. when the containers are torn down.
145+
146+
```sh
147+
make test-python-integration-local
148+
```
147149

148150
#### Full integration tests
149151
To test across clouds, on top of setting up Redis, you also need GCP / AWS / Snowflake setup.
@@ -166,7 +168,15 @@ To test across clouds, on top of setting up Redis, you also need GCP / AWS / Sno
166168
2. Modify `RedshiftDataSourceCreator` to use your credentials
167169

168170
**Snowflake**
169-
- See https://signup.snowflake.com/
171+
1. See https://signup.snowflake.com/ to setup a trial.
172+
2. Then to run successfully, you'll need some environment variables setup:
173+
```sh
174+
export SNOWFLAKE_CI_DEPLOYMENT='[snowflake_deployment]'
175+
export SNOWFLAKE_CI_USER='[your user]'
176+
export SNOWFLAKE_CI_PASSWORD='[your pw]'
177+
export SNOWFLAKE_CI_ROLE='[your CI role e.g. SYSADMIN]'
178+
export SNOWFLAKE_CI_WAREHOUSE='[your warehouse]'
179+
```
170180

171181
Then run `make test-python-integration`. Note that for Snowflake / GCP / AWS, this will create new temporary tables / datasets.
172182

‎Makefile‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,26 @@ test-python:
6868
test-python-integration:
6969
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
7070

71+
test-python-integration-local:
72+
@(docker info > /dev/null 2>&1 && \
73+
FEAST_USAGE=False \
74+
IS_TEST=True \
75+
FEAST_IS_LOCAL_TEST=True \
76+
FEAST_LOCAL_ONLINE_CONTAINER=True \
77+
python -m pytest -n 8 --integration \
78+
-k "not test_apply_entity_integration and \
79+
not test_apply_feature_view_integration and \
80+
not test_apply_data_source_integration" \
81+
sdk/python/tests \
82+
) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
83+
7184
test-python-integration-container:
72-
FEAST_USAGE=False IS_TEST=True FEAST_LOCAL_ONLINE_CONTAINER=True python -m pytest -n 8 --integration sdk/python/tests
85+
@(docker info > /dev/null 2>&1 && \
86+
FEAST_USAGE=False \
87+
IS_TEST=True \
88+
FEAST_LOCAL_ONLINE_CONTAINER=True \
89+
python -m pytest -n 8 --integration sdk/python/tests \
90+
) || echo "This script uses Docker, and it isn't running - please start the Docker Daemon and try again!";
7391

7492
test-python-universal-contrib:
7593
PYTHONPATH='.' \
@@ -104,14 +122,11 @@ test-python-universal-postgres:
104122
not test_universal_types" \
105123
sdk/python/tests
106124

107-
test-python-universal-local:
108-
FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration sdk/python/tests
109-
110125
test-python-universal:
111126
FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests
112127

113128
test-python-go-server: compile-go-lib
114-
FEAST_USAGE=False IS_TEST=True FEAST_GO_FEATURE_RETRIEVAL=True pytest --integration --goserver sdk/python/tests
129+
FEAST_USAGE=False IS_TEST=True pytest --integration --goserver sdk/python/tests
115130

116131
format-python:
117132
# Sort

0 commit comments

Comments
 (0)