Skip to content

Commit 71431f9

Browse files
aslisabanciaslisabanci
committed
Initial commit
0 parents  commit 71431f9

9 files changed

Lines changed: 527 additions & 0 deletions

File tree

‎.gitignore‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# C extensions
7+
*.so
8+
9+
# Distribution / packaging
10+
.Python
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
wheels/
23+
pip-wheel-metadata/
24+
share/python-wheels/
25+
*.egg-info/
26+
.installed.cfg
27+
*.egg
28+
MANIFEST
29+
30+
# PyInstaller
31+
# Usually these files are written by a python script from a template
32+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
33+
*.manifest
34+
*.spec
35+
36+
# Installer logs
37+
pip-log.txt
38+
pip-delete-this-directory.txt
39+
40+
# Unit test / coverage reports
41+
htmlcov/
42+
.tox/
43+
.nox/
44+
.coverage
45+
.coverage.*
46+
.cache
47+
nosetests.xml
48+
coverage.xml
49+
*.cover
50+
*.py,cover
51+
.hypothesis/
52+
.pytest_cache/
53+
54+
# Translations
55+
*.mo
56+
*.pot
57+
58+
# Django stuff:
59+
*.log
60+
local_settings.py
61+
db.sqlite3
62+
db.sqlite3-journal
63+
64+
# Flask stuff:
65+
instance/
66+
.webassets-cache
67+
68+
# Scrapy stuff:
69+
.scrapy
70+
71+
# Sphinx documentation
72+
docs/_build/
73+
74+
# PyBuilder
75+
target/
76+
77+
# Jupyter Notebook
78+
.ipynb_checkpoints
79+
80+
# IPython
81+
profile_default/
82+
ipython_config.py
83+
84+
# pyenv
85+
.python-version
86+
87+
# pipenv
88+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
90+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
91+
# install all needed dependencies.
92+
#Pipfile.lock
93+
94+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
95+
__pypackages__/
96+
97+
# Celery stuff
98+
celerybeat-schedule
99+
celerybeat.pid
100+
101+
# SageMath parsed files
102+
*.sage.py
103+
104+
# Environments
105+
.env
106+
.venv
107+
env/
108+
venv/
109+
ENV/
110+
env.bak/
111+
venv.bak/
112+
113+
# Spyder project settings
114+
.spyderproject
115+
.spyproject
116+
117+
# Rope project settings
118+
.ropeproject
119+
120+
# mkdocs documentation
121+
/site
122+
123+
# mypy
124+
.mypy_cache/
125+
.dmypy.json
126+
dmypy.json
127+
128+
# Pyre type checker
129+
.pyre/
130+
131+
# VSCode
132+
.vscode/
133+
134+
# DS_Store
135+
.DS_Store

‎Dockerfile‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from ubuntu:20.04
2+
3+
RUN apt-get update && apt-get install -y \
4+
git \
5+
python3.7 \
6+
python3-setuptools \
7+
python3-pip
8+
9+
RUN pip3 install algorithmia&& \
10+
pip3 install algorithmia-api-client&& \
11+
pip3 install jupyter&& \
12+
pip3 install nbformat&& \
13+
pip3 install nbconvert[execute]
14+
15+
COPY src /src
16+
COPY entrypoint.sh /entrypoint.sh
17+
RUN chmod +x entrypoint.sh
18+
ENTRYPOINT ["/entrypoint.sh"]

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Algorithmia CI - Model Deployment
2+
Github action to run a notebook and conditionally deploy the output model to Algorithmia [WIP]

‎action.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: 'Deploy to Algorithmia'
2+
description: 'TBD'
3+
branding:
4+
icon: 'alert-circle'
5+
color: 'yellow'
6+
inputs:
7+
algorithmia_api_key:
8+
description: 'Algorithmia API Key, used when uploading the model file'
9+
required: true
10+
algorithmia_username:
11+
description: 'Algorithmia Username, used as Git Commit Username and as part of default model upload path'
12+
required: true
13+
algorithmia_email:
14+
description: 'Algorithmia Email, used as Git Commit Email'
15+
required: true
16+
algorithmia_password:
17+
description: 'Algorithmia Password. Only required if the algorithm is hosted on Algorithmia.'
18+
required: false
19+
github_username:
20+
description: 'Github username owning the algorithm repository, used to clone the algorithm repository. Only required if the algorithm is hosted on Github.'
21+
required: false
22+
github_pat:
23+
description: 'Github Personal Access Token, used to clone the algorithm repository. Only required if the algorithm is hosted on Github.'
24+
required: false
25+
algorithmia_algoname:
26+
description: 'Algorithmia Algorithm name'
27+
required: true
28+
algorithmia_uploadpath:
29+
description: 'Algorithmia data collection name to upload the model file'
30+
required: true
31+
default: 'data://$ALGORITHMIA_USERNAME/$ALGORITHMIA_ALGONAME'
32+
model_path:
33+
description: 'Path of the model file to be uploaded to Algorithmia'
34+
required: true
35+
default: 'model.pkl'
36+
notebook_path:
37+
description: 'Path of the notebook to be executed by this action'
38+
required: false
39+
default: 'model_training.ipynb'
40+
git_host:
41+
description: Git host for the Algorithmia algorithm repository. Change to git.algorithmia.com if the algorithm is hosted on Algorithmia.
42+
required: false
43+
default: 'github.com'
44+
runs:
45+
using: 'docker'
46+
image: 'Dockerfile'
47+
args:
48+
- ${{ inputs.algorithmia_api_key }}
49+
- ${{ inputs.algorithmia_username }}
50+
- ${{ inputs.algorithmia_email }}
51+
- ${{ inputs.algorithmia_algoname }}
52+
- ${{ inputs.algorithmia_password }}
53+
- ${{ inputs.github_username }}
54+
- ${{ inputs.github_pat }}
55+
- ${{ inputs.algorithmia_uploadpath }}
56+
- ${{ inputs.model_path }}
57+
- ${{ inputs.notebook_path }}
58+
- ${{ inputs.git_host }}
59+
60+

‎entrypoint.sh‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
# Install the dependencies of this repo to be able to run the notebook
4+
pip3 install -r requirements.txt
5+
6+
# Clone the Algorithmia algorithm repository
7+
CI_ALGO_DIR=$INPUT_ALGORITHMIA_ALGONAME"_CI"
8+
9+
if [ -z "$INPUT_ALGORITHMIA_PASSWORD" ]
10+
then
11+
echo "Will clone algorithm repository hosted on Github"
12+
git clone https://"$INPUT_GITHUB_PAT"@"$INPUT_GIT_HOST"/"$INPUT_GITHUB_USERNAME"/"$INPUT_ALGORITHMIA_ALGONAME".git $CI_ALGO_DIR
13+
else
14+
echo "Will clone algorithm repository hosted on Algorithmia"
15+
git clone https://"$INPUT_ALGORITHMIA_USERNAME":"$INPUT_ALGORITHMIA_PASSWORD"@"$INPUT_GIT_HOST"/git/"$INPUT_ALGORITHMIA_USERNAME"/"$INPUT_ALGORITHMIA_ALGONAME".git $CI_ALGO_DIR
16+
fi
17+
18+
19+
20+
# Run action main to:
21+
# - (Optionally) run the notebook file
22+
# - Upload the model file to Algorithmia
23+
# - Link the algorithm with the uploaded model
24+
python3 /src/action_main.py
25+
26+
# Push updates to Algorithmia, and trigger a new algorithm build
27+
if [ $? -eq 0 ]
28+
then
29+
echo "Successfully executed action script, optionally executing the model notebook and uploading the model file to Algorithmia."
30+
31+
if ! [ -e $INPUT_ALGORITHMIA_ALGONAME.py ]
32+
then
33+
echo "$INPUT_ALGORITHMIA_ALGONAME.py does not exist. Checking if $INPUT_ALGORITHMIA_ALGONAME.ipynb exists."
34+
if [ -e $INPUT_ALGORITHMIA_ALGONAME.ipynb ]
35+
then
36+
echo "Found $INPUT_ALGORITHMIA_ALGONAME.ipynb. Will convert this into .py before pushing it to Algorithmia."
37+
jupyter nbconvert --to script $INPUT_ALGORITHMIA_ALGONAME.ipynb
38+
else
39+
echo "Neither $INPUT_ALGORITHMIA_ALGONAME.ipynb or $INPUT_ALGORITHMIA_ALGONAME.py exist."
40+
fi
41+
fi
42+
43+
if [ -e $INPUT_ALGORITHMIA_ALGONAME.py ]
44+
then
45+
echo "Will copy and push $INPUT_ALGORITHMIA_ALGONAME.py to Algorithm repository."
46+
cp -R "$INPUT_ALGORITHMIA_ALGONAME.py" $CI_ALGO_DIR/src/
47+
cp -R requirements.txt $CI_ALGO_DIR/requirements.txt
48+
else
49+
echo "Will not copy any inference code to Algorithmia."
50+
fi
51+
52+
cd $CI_ALGO_DIR
53+
git config --global user.name "$INPUT_ALGORITHMIA_USERNAME"
54+
git config --global user.email "$INPUT_ALGORITHMIA_EMAIL"
55+
git add .
56+
git commit -m "Automated deployment via Github CI"
57+
git push
58+
else
59+
echo "Action script exited with error." >&2
60+
exit 1
61+
fi
62+

‎src/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎src/action_main.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/python3
2+
3+
import os
4+
import algorithmia_deployer
5+
import notebook_executor
6+
7+
8+
if __name__ == "__main__":
9+
workspace = os.getenv("GITHUB_WORKSPACE")
10+
git_repo = os.getenv("GITHUB_REPOSITORY")
11+
git_ref = os.getenv("GITHUB_REF")
12+
commit_SHA = os.getenv("GITHUB_SHA")
13+
commit_msg = os.getenv("HEAD_COMMIT_MSG")
14+
algorithmia_api_key = os.getenv("INPUT_ALGORITHMIA_API_KEY")
15+
algorithmia_username = os.getenv("INPUT_ALGORITHMIA_USERNAME")
16+
algorithmia_algo_name = os.getenv("INPUT_ALGORITHMIA_ALGONAME")
17+
notebook_path = os.getenv("INPUT_NOTEBOOK_PATH")
18+
model_path = os.getenv("INPUT_MODEL_PATH")
19+
upload_path = os.getenv("INPUT_ALGORITHMIA_UPLOADPATH")
20+
21+
# TODO: Make github host parametric
22+
23+
error_template_str = "Field '{}' not defined in workflow file. Please check your workflow configuration"
24+
if not algorithmia_api_key:
25+
raise Exception(error_template_str.format("algorithmia_api_key"))
26+
# TODO: Add new inputs checks too
27+
if not model_path:
28+
raise Exception(error_template_str.format("model_path"))
29+
if not upload_path:
30+
raise Exception(error_template_str.format("algorithmia_uploadpath"))
31+
32+
if os.path.exists(workspace):
33+
try:
34+
notebook_executor = notebook_executor.NotebookExecutor(
35+
workspace, notebook_path
36+
)
37+
notebook_executor.run()
38+
39+
algorithmia_deployer = algorithmia_deployer.AlgorithmiaDeployer(
40+
api_key=algorithmia_api_key,
41+
username=algorithmia_username,
42+
algo_name=algorithmia_algo_name,
43+
model_path=model_path,
44+
workspace_path=workspace,
45+
)
46+
algorithmia_deployer.upload_and_link_algo_model(
47+
upload_path=upload_path,
48+
git_repo=git_repo,
49+
git_ref=git_ref,
50+
commit_SHA=commit_SHA,
51+
commit_msg=commit_msg,
52+
)
53+
except Exception as e:
54+
raise e
55+
else:
56+
raise Exception(
57+
"actions/checkout action should be run on the main repository, before this action. Please check your workflow configuration."
58+
)

0 commit comments

Comments
 (0)