20 changed files with 0 additions and 479 deletions
@ -1,62 +0,0 @@ |
|||
# Zcash CI workers |
|||
|
|||
This folder contains the Ansible playbooks for configuring a fresh OS |
|||
installation for use as a Buildbot worker in Zcash's CI. |
|||
|
|||
# Criteria for Adding Workers |
|||
|
|||
a. Don't add workers until users complain about a problem on a platform that |
|||
doesn't yet have workers. However, if we anticipate many users will use a |
|||
platform, we may pre-emptively add an unsupported worker for it. |
|||
|
|||
b. Prioritize the platforms that seem to have the most users. |
|||
|
|||
c. When adding workers, start by adding workers for the "most common" variant of |
|||
any distro. Then if users later encounter problems with a sub-variant, we can |
|||
consider adding new workers at that point. Example: add Ubuntu Desktop before |
|||
Xubuntu, on the assumption the former has a larger population base, and the |
|||
latter only materially differs in the GUI. |
|||
|
|||
# Setting up a latent worker on Amazon EC2 |
|||
|
|||
1. Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and |
|||
deploy the changes. |
|||
- This enables the Ansible playbook to run to completion, ending in the worker |
|||
connecting to the master. |
|||
|
|||
2. Start a basic EC2 instance using the template AMI for the target OS. |
|||
- Choose the smallest instance size, it won't be used for building Zcash. |
|||
|
|||
3. Figure out which user to log into the instance with. |
|||
- E.g. for the Ubuntu template, use "ubuntu" instead of "root" |
|||
- If you get an Ansible error later with a message like "Failed to connect to |
|||
the host via ssh: Received message too long 1349281121\r\n", that means the |
|||
instance is sending a text string in response to the SSH connection, and the |
|||
Ansible protocol is balking. Try manually logging in with the same |
|||
credentials to diagnose. |
|||
|
|||
4. Create `inventory/hosts` containing the following: |
|||
|
|||
[zcash-ci-worker-unix] |
|||
some-name ansible_host=<INSTANCE_IP> ansible_ssh_user=<USERNAME> |
|||
|
|||
5. Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`, |
|||
passing in the worker's Buildbot name and password. |
|||
- After a successful run, the worker should be connected to dev-ci.z.cash and |
|||
visible in its worker list. |
|||
|
|||
6. Create an AMI from the instance. This is the worker AMI to put into the |
|||
master.cfg for dev-ci.z.cash. |
|||
- 16 GB of storage should be sufficient. |
|||
|
|||
7. SSH into the instance, and edit the worker config to connect to ci.z.cash. |
|||
|
|||
8. Create an AMI from the instance. This is the worker AMI to put into the |
|||
master.cfg for ci.z.cash. |
|||
- 16 GB of storage should be sufficient. |
|||
|
|||
9. Delete the instance (it is no longer needed). |
|||
|
|||
10. Edit the master.cfg to turn the new worker into a latent (using the new AMI |
|||
IDs), add it to the appropriate worker groups, set up new builders etc. |
|||
- Deploy this via the normal PR review process. |
|||
@ -1,2 +0,0 @@ |
|||
[ssh_connection] |
|||
pipelining = True |
|||
@ -1,2 +0,0 @@ |
|||
export PATH=$HOME/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin |
|||
export EDITOR=vim |
|||
@ -1,27 +0,0 @@ |
|||
--- |
|||
# Configure a Buildbot worker |
|||
- include: unix.yml |
|||
|
|||
- name: Install grind-specific worker dependencies |
|||
hosts: zcash-ci-worker-unix |
|||
become: true |
|||
|
|||
vars_files: |
|||
- vars/default.yml |
|||
|
|||
tasks: |
|||
- name: Get dependencies for distribution |
|||
include_vars: "{{ item }}" |
|||
with_first_found: |
|||
- files: |
|||
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml" |
|||
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml" |
|||
- "vars/{{ ansible_distribution }}.yml" |
|||
- "vars/{{ ansible_os_family }}.yml" |
|||
skip: true |
|||
|
|||
- name: Install required packages |
|||
package: |
|||
name: "{{ item }}" |
|||
state: present |
|||
with_items: "{{ grind_deps }}" |
|||
@ -1,10 +0,0 @@ |
|||
--- |
|||
- name: Check if brew is installed |
|||
stat: |
|||
path: /usr/local/bin/brew |
|||
register: brew_check |
|||
|
|||
- name: Fail if brew is unavailable |
|||
fail: |
|||
msg: 'brew is not installed! Please install Homebrew: https://docs.brew.sh/Installation.html' |
|||
when: not brew_check.stat.exists |
|||
@ -1,8 +0,0 @@ |
|||
--- |
|||
- name: Fetch pip installer |
|||
get_url: |
|||
url: https://bootstrap.pypa.io/get-pip.py |
|||
dest: /tmp/get-pip.py |
|||
|
|||
- name: Install pip |
|||
command: "{{ ansible_python.executable }} /tmp/get-pip.py" |
|||
@ -1,23 +0,0 @@ |
|||
<?xml version="1.0" encoding="UTF-8"?> |
|||
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> |
|||
<plist version="1.0"> |
|||
<dict> |
|||
<key>Label</key> |
|||
<string>net.buildbot.worker</string> |
|||
<key>ProgramArguments</key> |
|||
<array> |
|||
<string>{{ buildbot_worker_dir }}/venv/bin/buildbot-worker</string> |
|||
<string>start</string> |
|||
<string>{{ buildbot_worker_name }}</string> |
|||
</array> |
|||
<key>WorkingDirectory</key> |
|||
<string>{{ buildbot_worker_dir }}</string> |
|||
<key>UserName</key> |
|||
<string>{{ buildbot_worker_user }}</string> |
|||
<key>KeepAlive</key> |
|||
<dict> |
|||
<key>NetworkState</key> |
|||
<true/> |
|||
</dict> |
|||
</dict> |
|||
</plist> |
|||
@ -1,17 +0,0 @@ |
|||
[Unit] |
|||
Description=Buildbot worker |
|||
Wants=network.target |
|||
After=network.target |
|||
|
|||
[Service] |
|||
Type=forking |
|||
PIDFile={{ buildbot_worker_dir }}/{{ buildbot_worker_name }}/twistd.pid |
|||
WorkingDirectory={{ buildbot_worker_dir }} |
|||
ExecStart={{ buildbot_worker_dir }}/venv/bin/buildbot-worker start {{ buildbot_worker_name }} |
|||
ExecReload={{ buildbot_worker_dir }}/venv/bin/buildbot-worker restart {{ buildbot_worker_name }} |
|||
ExecStop={{ buildbot_worker_dir }}/venv/bin/buildbot-worker stop {{ buildbot_worker_name }} |
|||
Restart=always |
|||
User={{ buildbot_worker_user }} |
|||
|
|||
[Install] |
|||
WantedBy=multi-user.target |
|||
@ -1 +0,0 @@ |
|||
OS: {{ ansible_distribution }} {{ ansible_distribution_version }} |
|||
@ -1,3 +0,0 @@ |
|||
OS: {{ ansible_distribution }} {{ ansible_distribution_version }} |
|||
Memory: {{ ansible_memtotal_mb }} MB |
|||
CPU: {{ ansible_processor if ansible_processor is string else ansible_processor[1] }} ({{ ansible_processor_cores }} cores) |
|||
@ -1,209 +0,0 @@ |
|||
--- |
|||
- name: Configure a Buildbot worker for Zcash CI |
|||
hosts: zcash-ci-worker-unix |
|||
become: true |
|||
gather_facts: False |
|||
|
|||
vars_files: |
|||
- vars/default.yml |
|||
- vars/buildbot.yml |
|||
|
|||
vars_prompt: |
|||
- name: "buildbot_worker_admin" |
|||
prompt: "Admin details" |
|||
default: "Zcash <sysadmin@z.cash>" |
|||
- name: "buildbot_worker_name" |
|||
prompt: "Buildbot worker name (provided by ZECC)" |
|||
private: no |
|||
- name: "buildbot_worker_password" |
|||
prompt: "Buildbot worker password (provided by ZECC)" |
|||
|
|||
pre_tasks: |
|||
- name: Install Python 2.7 for Ansible and Buildbot |
|||
raw: test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2) |
|||
register: output |
|||
changed_when: |
|||
- output.stdout != "" |
|||
- output.stdout != "\r\n" |
|||
|
|||
- name: Check if Python is in the configured location |
|||
raw: test -e {{ ansible_python_interpreter }} |
|||
ignore_errors: true |
|||
register: python_check |
|||
when: ansible_python_interpreter is defined |
|||
|
|||
- name: Fail if configured Python is unavailable |
|||
fail: |
|||
msg: "Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary." |
|||
when: ansible_python_interpreter is defined and python_check.rc == 1 |
|||
|
|||
- name: Check if Python is in the default location |
|||
raw: test -e /usr/bin/python |
|||
ignore_errors: true |
|||
register: python_check |
|||
when: ansible_python_interpreter is undefined |
|||
|
|||
- name: Fail if default Python is unavailable |
|||
fail: |
|||
msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary. |
|||
when: ansible_python_interpreter is undefined and python_check.rc == 1 |
|||
|
|||
- name: Gathering Facts |
|||
setup: |
|||
tags: deps |
|||
|
|||
- name: Fail if Python is the wrong version |
|||
fail: |
|||
msg: "The Python binary at {{ ansible_python.executable }} is version {{ ansible_python_version }}! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary." |
|||
when: ansible_python.version.major != 2 or ansible_python.version.minor != 7 |
|||
|
|||
tasks: |
|||
- name: Get dependencies for distribution |
|||
include_vars: "{{ item }}" |
|||
with_first_found: |
|||
- files: |
|||
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml" |
|||
- "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml" |
|||
- "vars/{{ ansible_distribution }}.yml" |
|||
- "vars/{{ ansible_os_family }}.yml" |
|||
skip: true |
|||
tags: deps |
|||
|
|||
- name: Collate dependencies |
|||
set_fact: |
|||
package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}" |
|||
python_modules: "{{ buildbot_modules + rpc_test_modules }}" |
|||
tags: deps |
|||
|
|||
- name: Install Homebrew [MacOSX] |
|||
include: tasks/install-brew.yml |
|||
when: ansible_distribution == 'MacOSX' |
|||
tags: deps |
|||
|
|||
- name: Update rolling release [Arch Linux] |
|||
pacman: |
|||
update_cache: yes |
|||
upgrade: yes |
|||
when: ansible_distribution == 'Archlinux' |
|||
tags: deps |
|||
|
|||
- name: Install required packages |
|||
package: |
|||
name: "{{ item }}" |
|||
state: present |
|||
with_items: "{{ package_deps }}" |
|||
become_user: "{{ ansible_ssh_user if ansible_distribution == 'MacOSX' else 'root' }}" |
|||
tags: deps |
|||
|
|||
- name: Install pip [CentOS, MacOSX] |
|||
include: tasks/install-pip.yml |
|||
when: ansible_distribution in ['CentOS', 'MacOSX'] |
|||
|
|||
- name: Install required Python system modules |
|||
pip: |
|||
name: "{{ item }}" |
|||
state: latest |
|||
executable: "{{ '/usr/local/bin/pip' if ansible_distribution == 'MacOSX' else omit }}" |
|||
with_items: "{{ system_modules }}" |
|||
|
|||
- name: Set up the Buildbot worker user |
|||
user: |
|||
name: "{{ buildbot_worker_user }}" |
|||
comment: Buildbot worker |
|||
shell: /bin/bash |
|||
state: present |
|||
|
|||
- name: Get absolute path to Buildbot worker home directory |
|||
command: echo ~ |
|||
register: homedir |
|||
become_user: "{{ buildbot_worker_user }}" |
|||
|
|||
- name: Save absolute path to Buildbot worker home directory |
|||
set_fact: |
|||
buildbot_worker_dir: "{{ homedir.stdout }}" |
|||
|
|||
- name: Install required Python modules |
|||
pip: |
|||
name: "{{ item }}" |
|||
state: latest |
|||
virtualenv: "~{{ buildbot_worker_user }}/venv" |
|||
virtualenv_command: "{{ '/usr/local/bin/virtualenv' if ansible_distribution == 'MacOSX' else omit }}" |
|||
with_items: "{{ python_modules }}" |
|||
become_user: "{{ buildbot_worker_user }}" |
|||
notify: restart buildbot-worker |
|||
|
|||
- name: Create Buildbot worker |
|||
command: > |
|||
~{{ buildbot_worker_user }}/venv/bin/buildbot-worker create-worker ~/{{ buildbot_worker_name }} |
|||
{{ buildbot_master_host }}:{{ buildbot_master_port }} |
|||
{{ buildbot_worker_name|quote }} {{ buildbot_worker_password|quote }} |
|||
args: |
|||
creates: "~/{{ buildbot_worker_name }}/buildbot.tac" |
|||
become_user: "{{ buildbot_worker_user }}" |
|||
|
|||
- name: Set admin details for Buildbot worker |
|||
copy: |
|||
content: "{{ buildbot_worker_admin }}" |
|||
dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/admin" |
|||
owner: "{{ buildbot_worker_user }}" |
|||
group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}" |
|||
mode: "0644" |
|||
|
|||
- name: Set host details for Buildbot worker |
|||
template: |
|||
src: "{{ buildbot_worker_host_template }}" |
|||
dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/host" |
|||
owner: "{{ buildbot_worker_user }}" |
|||
group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}" |
|||
mode: "0644" |
|||
|
|||
- name: Install custom bashrc for virtualenv |
|||
copy: |
|||
src: bashrc |
|||
dest: "~{{ buildbot_worker_user }}/.bashrc" |
|||
owner: "{{ buildbot_worker_user }}" |
|||
group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}" |
|||
mode: "0644" |
|||
|
|||
- name: Copy Buildbot worker systemd service unit |
|||
template: |
|||
src: templates/buildbot-worker.service.j2 |
|||
dest: "/etc/systemd/system/buildbot-worker.service" |
|||
owner: root |
|||
group: root |
|||
mode: "0644" |
|||
when: ansible_distribution != 'MacOSX' |
|||
notify: reload systemd |
|||
|
|||
- name: Copy Buildbot worker launchd service unit |
|||
template: |
|||
src: templates/buildbot-worker.plist.j2 |
|||
dest: "/Library/LaunchDaemons/net.buildbot.worker.plist" |
|||
owner: root |
|||
group: wheel |
|||
mode: "0644" |
|||
when: ansible_distribution == 'MacOSX' |
|||
|
|||
- name: Start Buildbot worker |
|||
service: |
|||
name: buildbot-worker |
|||
state: started |
|||
enabled: yes |
|||
when: ansible_distribution != 'MacOSX' |
|||
|
|||
- name: Load Buildbot worker service [MacOSX] |
|||
command: launchctl load /Library/LaunchDaemons/net.buildbot.worker.plist |
|||
when: ansible_distribution == 'MacOSX' |
|||
|
|||
- name: Start Buildbot worker [MacOSX] |
|||
command: launchctl start net.buildbot.worker |
|||
when: ansible_distribution == 'MacOSX' |
|||
|
|||
handlers: |
|||
- name: restart buildbot-worker |
|||
service: |
|||
name: buildbot-worker |
|||
state: restarted |
|||
|
|||
- name: reload systemd |
|||
command: /bin/systemctl daemon-reload |
|||
@ -1,7 +0,0 @@ |
|||
--- |
|||
buildbot_deps: |
|||
- python2-pip |
|||
build_deps: |
|||
- cmake |
|||
- multilib/gcc |
|||
- make |
|||
@ -1,13 +0,0 @@ |
|||
--- |
|||
buildbot_deps: [] # Empty to remove python-pip |
|||
build_deps: |
|||
- bzip2 |
|||
- cmake |
|||
- gcc |
|||
- gcc-c++ |
|||
- make |
|||
- patch |
|||
dist_deps: |
|||
- pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in |
|||
- python-devel |
|||
- redhat-rpm-config |
|||
@ -1,6 +0,0 @@ |
|||
--- |
|||
build_deps: |
|||
- build-essential # Depends on g++, libc6-dev, make |
|||
- cmake |
|||
dist_deps: |
|||
- python-dev |
|||
@ -1,11 +0,0 @@ |
|||
--- |
|||
build_deps: |
|||
- cmake |
|||
- gcc |
|||
- gcc-c++ |
|||
- make |
|||
- patch |
|||
dist_deps: |
|||
- pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in |
|||
- python-devel |
|||
- redhat-rpm-config |
|||
@ -1,10 +0,0 @@ |
|||
--- |
|||
buildbot_deps: |
|||
- py27-pip |
|||
build_deps: |
|||
- cmake |
|||
- gcc |
|||
- gmake |
|||
dist_deps: |
|||
- bash |
|||
- pkgconf # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in |
|||
@ -1,6 +0,0 @@ |
|||
--- |
|||
buildbot_deps: |
|||
- coreutils # For gnproc etc. |
|||
# Most are already installed |
|||
build_deps: |
|||
- cmake |
|||
@ -1,4 +0,0 @@ |
|||
--- |
|||
build_deps: |
|||
- build-essential # Depends on g++, libc6-dev, make |
|||
- cmake |
|||
@ -1,5 +0,0 @@ |
|||
--- |
|||
buildbot_worker_user: zcbbworker |
|||
buildbot_master_host: dev-ci.z.cash |
|||
buildbot_master_port: 9899 |
|||
buildbot_worker_host_template: templates/host.j2 |
|||
@ -1,53 +0,0 @@ |
|||
--- |
|||
# These variables can be overridden in distribution files. |
|||
|
|||
# Dependencies required to install Buildbot |
|||
buildbot_deps: |
|||
- python-pip # So we can install Python modules |
|||
|
|||
# Dependencies required to download files |
|||
fetch_deps: |
|||
- curl # For depends/ |
|||
- git |
|||
- wget # For zcutil/fetch-params.sh |
|||
|
|||
# Dependencies required to configure Zcash |
|||
conf_deps: |
|||
- autoconf |
|||
- automake |
|||
- m4 |
|||
- pkg-config |
|||
|
|||
# Dependencies required to compile Zcash |
|||
build_deps: |
|||
- cmake |
|||
- g++ |
|||
- gcc |
|||
- make |
|||
|
|||
# Dependencies required to link Zcash |
|||
link_deps: |
|||
- libtool |
|||
|
|||
# Additional distribution-specific dependencies |
|||
dist_deps: [] |
|||
|
|||
# Additional grind-specific dependencies |
|||
grind_deps: |
|||
- lcov |
|||
- valgrind |
|||
|
|||
# Python modules required on the system |
|||
system_modules: |
|||
- virtualenv |
|||
|
|||
# Python modules required for a Zcash Buildbot worker |
|||
buildbot_modules: |
|||
- pip # Needs to be updated first so Buildbot installs |
|||
- buildbot-worker |
|||
- pyflakes |
|||
|
|||
# Python modules required to run the Zcash RPC test suite |
|||
rpc_test_modules: |
|||
- pyblake2 |
|||
- pyzmq |
|||
Loading…
Reference in new issue