Auto merge of #2560 - str4d:ci-workers-bugfix, r=str4d

ci-workers: Bugfixes and improvements

contrib/ci-workers/ansible.cfg

@@ -0,0 +1,2 @@
+[ssh_connection]
+pipelining = True

contrib/ci-workers/tasks/install-pip.yml

@@ -0,0 +1,8 @@
+---
+- name: Fetch pip installer
+  get_url:
+    url: https://bootstrap.pypa.io/get-pip.py
+    dest: /tmp/get-pip.py
+
+- name: Install pip
+  command: "{{ ansible_python.executable }} /tmp/get-pip.py"

contrib/ci-workers/templates/buildbot-worker.service.j2

@@ -7,9 +7,9 @@ After=network.target
 Type=forking
 PIDFile=/home/{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/twistd.pid
 WorkingDirectory=/home/{{ buildbot_worker_user }}
-ExecStart=/usr/local/bin/buildbot-worker start {{ buildbot_worker_name }}
-ExecReload=/usr/local/bin/buildbot-worker restart {{ buildbot_worker_name }}
-ExecStop=/usr/local/bin/buildbot-worker stop {{ buildbot_worker_name }}
+ExecStart={{ pip_bin_dir }}/buildbot-worker start {{ buildbot_worker_name }}
+ExecReload={{ pip_bin_dir }}/buildbot-worker restart {{ buildbot_worker_name }}
+ExecStop={{ pip_bin_dir }}/buildbot-worker stop {{ buildbot_worker_name }}
 Restart=always
 User={{ buildbot_worker_user }}
 

contrib/ci-workers/unix.yml

@@ -19,7 +19,7 @@
       prompt: "Buildbot worker password (provided by ZECC)"
 
   pre_tasks:
-    - name: Install Python 2 for Ansible and Buildbot
+    - name: Install Python 2.7 for Ansible and Buildbot
       raw: test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2)
       register: output
       changed_when:
@@ -34,7 +34,7 @@
 
     - name: Fail if configured Python is unavailable
       fail:
-        msg: Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
+        msg: "Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary."
       when: ansible_python_interpreter is defined and python_check.rc == 1
 
     - name: Check if Python is in the default location
@@ -45,12 +45,17 @@
 
     - name: Fail if default Python is unavailable
       fail:
-        msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2 binary.
+        msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary.
       when: ansible_python_interpreter is undefined and python_check.rc == 1
 
     - name: Gathering Facts
       setup:
 
+    - name: Fail if Python is the wrong version
+      fail:
+        msg: "The Python binary at {{ ansible_python.executable }} is version {{ ansible_python_version }}! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary."
+      when: ansible_python.version.major != 2 or ansible_python.version.minor != 7
+
   tasks:
     - name: Get dependencies for distribution
       include_vars: "{{ item }}"
@@ -67,12 +72,22 @@
         package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}"
         python_modules: "{{ buildbot_modules + rpc_test_modules }}"
 
+    - name: Update rolling release [Arch Linux]
+      pacman:
+        update_cache: yes
+        upgrade: yes
+      when: ansible_distribution == 'Archlinux'
+
     - name: Install required packages
       package:
         name: "{{ item }}"
         state: present
       with_items: "{{ package_deps }}"
 
+    - name: Install pip [CentOS]
+      include: tasks/install-pip.yml
+      when: ansible_distribution == 'CentOS'
+
     - name: Install required Python modules
       pip:
         name: "{{ item }}"
@@ -99,14 +114,18 @@
     - name: Set admin details for Buildbot worker
       copy:
         content: "{{ buildbot_worker_admin }}"
-        dest: "~/{{ buildbot_worker_name }}/info/admin"
-      become_user: "{{ buildbot_worker_user }}"
+        dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/admin"
+        owner: "{{ buildbot_worker_user }}"
+        group: "{{ buildbot_worker_user }}"
+        mode: "0644"
 
     - name: Set host details for Buildbot worker
       template:
         src: "{{ buildbot_worker_host_template }}"
-        dest: "~/{{ buildbot_worker_name }}/info/host"
-      become_user: "{{ buildbot_worker_user }}"
+        dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/host"
+        owner: "{{ buildbot_worker_user }}"
+        group: "{{ buildbot_worker_user }}"
+        mode: "0644"
 
     - name: Copy Buildbot worker systemd service unit
       template:

contrib/ci-workers/vars/Archlinux.yml

@@ -0,0 +1,7 @@
+---
+buildbot_deps:
+  - python2-pip
+build_deps:
+  - multilib/gcc
+  - make
+pip_bin_dir: /usr/bin

contrib/ci-workers/vars/CentOS.yml

@@ -0,0 +1,13 @@
+---
+buildbot_deps: [] # Empty to remove python-pip
+build_deps:
+  - bzip2
+  - gcc
+  - gcc-c++
+  - make
+  - patch
+dist_deps:
+  - pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in
+  - python-devel
+  - redhat-rpm-config
+pip_bin_dir: /usr/bin

contrib/ci-workers/vars/default.yml

@@ -39,3 +39,6 @@ buildbot_modules:
 rpc_test_modules:
   - pyblake2
   - pyzmq
+
+# Environment variables
+pip_bin_dir: /usr/local/bin