From 85fe644139e24bb24d385ed018ca3f204b2cca5f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 15 Oct 2016 13:47:10 -0500 Subject: [PATCH 01/18] Remaining changes from bitcoin/bitcoin#6854 Add a check to symbol-check.py that checks that only the subset of allowed libraries is imported (to avoid incompatibilities). See 56734f4b27eef8727bfbaa017b88fb30caddb7ab for the earlier changes. --- contrib/devtools/symbol-check.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index fad891f80..34f1ed2d1 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # Copyright (c) 2014 Wladimir J. van der Laan # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,6 +15,7 @@ from __future__ import division, print_function import subprocess import re import sys +import os # Debian 6.0.9 (Squeeze) has: # @@ -45,8 +46,10 @@ MAX_VERSIONS = { IGNORE_EXPORTS = { '_edata', '_end', '_init', '__bss_start', '_fini' } -READELF_CMD = '/usr/bin/readelf' -CPPFILT_CMD = '/usr/bin/c++filt' +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +# Allowed NEEDED libraries +ALLOWED_LIBRARIES = {'librt.so.1','libpthread.so.0','libanl.so.1','libm.so.6','libgcc_s.so.1','libc.so.6','ld-linux-x86-64.so.2'} class CPPFilt(object): ''' @@ -98,6 +101,22 @@ def check_version(max_versions, version): return False return ver <= max_versions[lib] +def read_libraries(filename): + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.split('\n'): + tokens = line.split() + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + if match: + libraries.append(match.group(1)) + else: + raise ValueError('Unparseable (NEEDED) specification') + return libraries + if __name__ == '__main__': cppfilt = CPPFilt() retval = 0 @@ -113,6 +132,11 @@ if __name__ == '__main__': continue print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) retval = 1 + # Check dependency libraries + for library_name in read_libraries(filename): + if library_name not in ALLOWED_LIBRARIES: + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + retval = 1 exit(retval) From 3599031edc36ae455309de0b025885ccc4851adf Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 28 Oct 2015 18:08:53 +0100 Subject: [PATCH 02/18] devtools: add libraries for bitcoin-qt to symbol check Forgot to add these. Also add a short description for each required library. --- contrib/devtools/symbol-check.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 34f1ed2d1..93acfcdda 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -49,7 +49,24 @@ IGNORE_EXPORTS = { READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries -ALLOWED_LIBRARIES = {'librt.so.1','libpthread.so.0','libanl.so.1','libm.so.6','libgcc_s.so.1','libc.so.6','ld-linux-x86-64.so.2'} +ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libanl.so.1', # DNS resolve +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +# bitcoin-qt only +'libX11-xcb.so.1', # part of X11 +'libX11.so.6', # part of X11 +'libxcb.so.1', # part of X11 +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2' # programming interface to dynamic linker +} class CPPFilt(object): ''' From bdb3f2ce3ef296f4977c2f179074408900072876 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 29 Oct 2015 07:29:48 +0100 Subject: [PATCH 03/18] gitian: use trusty for building --- contrib/gitian-descriptors/README.md | 2 +- contrib/gitian-descriptors/gitian-linux.yml | 17 ++++------------- .../gitian-descriptors/gitian-osx-signer.yml | 4 ++-- contrib/gitian-descriptors/gitian-osx.yml | 8 ++++---- .../gitian-descriptors/gitian-win-signer.yml | 2 +- contrib/gitian-descriptors/gitian-win.yml | 8 ++++---- 6 files changed, 16 insertions(+), 25 deletions(-) diff --git a/contrib/gitian-descriptors/README.md b/contrib/gitian-descriptors/README.md index 061b897d2..07c2ba98b 100644 --- a/contrib/gitian-descriptors/README.md +++ b/contrib/gitian-descriptors/README.md @@ -27,7 +27,7 @@ Once you've got the right hardware and software: # Create base images cd gitian-builder - bin/make-base-vm --suite precise --arch amd64 + bin/make-base-vm --suite trusty --arch amd64 cd .. # Get inputs (see doc/release-process.md for exact inputs needed and where to get them) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index d4df9eceb..1b67a43f0 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -2,20 +2,19 @@ name: "bitcoin-linux-0.11" enable_cache: true suites: -- "precise" +- "trusty" architectures: - "amd64" packages: - "g++-multilib" - "git-core" - "pkg-config" -- "autoconf2.13" +- "autoconf" - "libtool" - "automake" - "faketime" - "bsdmainutils" - "binutils-gold" -- "libstdc++6-4.6-pic" reference_datetime: "2015-06-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" @@ -44,7 +43,7 @@ script: | for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -55,7 +54,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} @@ -70,14 +69,6 @@ script: | make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done - # Ubuntu precise hack: Not an issue in later versions. - # Precise's libstdc++.a is non-pic. There's an optional libstdc++6-4.6-pic - # package which provides libstdc++_pic.a, but the linker can't find it. - # Symlink it to a path that will be included in our link-line so that the - # linker picks it up before the default libstdc++.a. - # This is only necessary for 64bit. - ln -s /usr/lib/gcc/x86_64-linux-gnu/4.6/libstdc++_pic.a ${BASEPREFIX}/x86_64-unknown-linux-gnu/lib/libstdc++.a - # Create the release tarball using (arbitrarily) the first host ./autogen.sh ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 36d7b0126..aa9494b7e 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -1,7 +1,7 @@ --- name: "bitcoin-dmg-signer" suites: -- "precise" +- "trusty" architectures: - "amd64" packages: @@ -23,7 +23,7 @@ script: | for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 47eaf647c..ff61e793c 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -2,14 +2,14 @@ name: "bitcoin-osx-0.11" enable_cache: true suites: -- "precise" +- "trusty" architectures: - "amd64" packages: - "g++" - "git-core" - "pkg-config" -- "autoconf2.13" +- "autoconf" - "libtool" - "automake" - "faketime" @@ -49,7 +49,7 @@ script: | for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -60,7 +60,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 2a73050e0..a29d7ab47 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -1,7 +1,7 @@ --- name: "bitcoin-win-signer" suites: -- "precise" +- "trusty" architectures: - "amd64" packages: diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 5b3aa41e8..d96cbbdeb 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -2,14 +2,14 @@ name: "bitcoin-win-0.11" enable_cache: true suites: -- "precise" +- "trusty" architectures: - "amd64" packages: - "g++" - "git-core" - "pkg-config" -- "autoconf2.13" +- "autoconf" - "libtool" - "automake" - "faketime" @@ -46,7 +46,7 @@ script: | for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} @@ -57,7 +57,7 @@ script: | for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} From db57ce9d3f265929d988e033201ead1b7d16180b Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 30 Oct 2015 18:19:09 +0100 Subject: [PATCH 04/18] gitian: make windows build deterministic --- contrib/gitian-descriptors/gitian-win.yml | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index d96cbbdeb..1ce7bcc9f 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -63,6 +63,30 @@ script: | chmod +x ${WRAP_DIR}/${i}-${prog} done done + + # Create per-host linker wrapper + # This is only needed for trusty, as the mingw linker leaks a few bytes of + # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 + for i in $HOSTS; do + mkdir -p ${WRAP_DIR}/${i} + for prog in collect2; do + echo '#!/bin/bash' > ${WRAP_DIR}/${i}/${prog} + REAL=$(${i}-gcc -print-prog-name=${prog}) + echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} + echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} + chmod +x ${WRAP_DIR}/${i}/${prog} + done + for prog in gcc g++; do + echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + export PATH=${WRAP_DIR}:${PATH} cd bitcoin From 752a0848f26bcb9fedd0f59149f62f920d64dfa4 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 25 Dec 2015 12:30:45 +0100 Subject: [PATCH 05/18] [gitian] Set reference date to something more recent --- contrib/gitian-descriptors/gitian-linux.yml | 2 +- contrib/gitian-descriptors/gitian-osx-signer.yml | 2 +- contrib/gitian-descriptors/gitian-osx.yml | 2 +- contrib/gitian-descriptors/gitian-win-signer.yml | 2 +- contrib/gitian-descriptors/gitian-win.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 1b67a43f0..db144b234 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -15,7 +15,7 @@ packages: - "faketime" - "bsdmainutils" - "binutils-gold" -reference_datetime: "2015-06-01 00:00:00" +reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index aa9494b7e..5b52c492f 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -7,7 +7,7 @@ architectures: packages: - "libc6:i386" - "faketime" -reference_datetime: "2015-06-01 00:00:00" +reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git" "dir": "signature" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index ff61e793c..f9879c261 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -18,7 +18,7 @@ packages: - "libcap-dev" - "libz-dev" - "libbz2-dev" -reference_datetime: "2015-06-01 00:00:00" +reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index a29d7ab47..27c4f01eb 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -7,7 +7,7 @@ architectures: packages: - "libssl-dev" - "autoconf" -reference_datetime: "2015-06-01 00:00:00" +reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git" "dir": "signature" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 1ce7bcc9f..4d9acb10a 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -18,7 +18,7 @@ packages: - "g++-mingw-w64" - "nsis" - "zip" -reference_datetime: "2015-06-01 00:00:00" +reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" From 5494f303a4bbd8847c5a9cc8c0dfba92a911e8c9 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 19 Nov 2015 13:25:08 +0100 Subject: [PATCH 06/18] gitian: Need `ca-certificates` and `python` for LXC builds [Zcash: removed doc/gitian-building.md] --- contrib/gitian-descriptors/gitian-linux.yml | 2 ++ contrib/gitian-descriptors/gitian-osx.yml | 2 ++ contrib/gitian-descriptors/gitian-win.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index db144b234..ec34cc551 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -15,6 +15,8 @@ packages: - "faketime" - "bsdmainutils" - "binutils-gold" +- "ca-certificates" +- "python" reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index f9879c261..664ca9dfe 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -18,6 +18,8 @@ packages: - "libcap-dev" - "libz-dev" - "libbz2-dev" +- "ca-certificates" +- "python" reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 4d9acb10a..72ff095b9 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -18,6 +18,8 @@ packages: - "g++-mingw-w64" - "nsis" - "zip" +- "ca-certificates" +- "python" reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" From 3797182d51211bfcbd3df05b61c75e3bf8312d87 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 26 Jan 2016 23:03:15 -0500 Subject: [PATCH 07/18] release: add _IO_stdin_used to ignored exports For details see: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 --- contrib/devtools/symbol-check.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 93acfcdda..4ad5136f7 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -42,9 +42,12 @@ MAX_VERSIONS = { 'GLIBCXX': (3,4,13), 'GLIBC': (2,11) } +# See here for a description of _IO_stdin_used: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 + # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '_init', '__bss_start', '_fini' +'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used' } READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') From a5e918a07f976c384e82d1fb8e5c785ee6cb0534 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 26 Jan 2016 14:52:56 -0500 Subject: [PATCH 08/18] release: add check-symbols and check-security make targets These are not added to the default checks because some of them depend on release-build configs. [Zcash: See 56734f4b27eef8727bfbaa017b88fb30caddb7ab for check-security] --- src/Makefile.am | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 28b73431d..e82754e00 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -79,7 +79,7 @@ LIBZCASH_H = \ zcash/prf.h \ zcash/util.h -.PHONY: FORCE check-security +.PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ addrman.h \ @@ -482,6 +482,12 @@ clean-local: $(AM_V_CXX) $(OBJCXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ $(CPPFLAGS) $(AM_CXXFLAGS) $(QT_INCLUDES) $(CXXFLAGS) -c -o $@ $< +check-symbols: $(bin_PROGRAMS) +if GLIBC_BACK_COMPAT + @echo "Checking glibc back compat of [$(bin_PROGRAMS)]..." + $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) +endif + check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security of [$(bin_PROGRAMS)]..." From d68e2111733720a634a1f2b039b682abcda6fe95 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 26 Jan 2016 22:36:39 -0500 Subject: [PATCH 09/18] release: always link librt for glibc back-compat builds glibc absorbed clock_gettime in 2.17. librt (its previous location) is safe to link in anyway for back-compat. Fixes #7420 --- configure.ac | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 715ed897b..81cb894e2 100644 --- a/configure.ac +++ b/configure.ac @@ -393,6 +393,10 @@ AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then + #glibc absorbed clock_gettime in 2.17. librt (its previous location) is safe to link + #in anyway for back-compat. + AC_CHECK_LIB([rt],[clock_gettime],, AC_MSG_ERROR(lib missing)) + #__fdelt_chk's params and return type have changed from long unsigned int to long int. # See which one is present here. AC_MSG_CHECKING(__fdelt_chk type) @@ -406,7 +410,8 @@ if test x$use_glibc_compat != xno; then [ fdelt_type="long int"]) AC_MSG_RESULT($fdelt_type) AC_DEFINE_UNQUOTED(FDELT_TYPE, $fdelt_type,[parameter and return value type for __fdelt_chk]) - +else + AC_SEARCH_LIBS([clock_gettime],[rt]) fi if test x$use_hardening != xno; then @@ -478,8 +483,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [ AC_MSG_RESULT(no)] ) -AC_SEARCH_LIBS([clock_gettime],[rt]) - AC_MSG_CHECKING([for visibility attribute]) AC_LINK_IFELSE([AC_LANG_SOURCE([ int foo_def( void ) __attribute__((visibility("default"))); From 8ef3b1b9340749bed9ce51815ba696773663a7bd Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 26 Jan 2016 15:00:30 -0500 Subject: [PATCH 10/18] release: add security/symbol checks to gitian --- contrib/gitian-descriptors/gitian-linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index ec34cc551..df8c829fb 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -97,6 +97,7 @@ script: | ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security + make ${MAKEOPTS} -C src check-symbols make install-strip cd installed find . -name "lib*.la" -delete From 2762638a01aa5aa22db5251492d03e4a184fb14e Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 31 Mar 2016 14:50:10 +0200 Subject: [PATCH 11/18] build: Remove unnecessary executables from gitian release This removes the following executables from the binary gitian release: - test_bitcoin-qt[.exe] - bench_bitcoin[.exe] @jonasschnelli and me discussed this on IRC a few days ago - unlike the normal `bitcoin_tests` which is useful to see if it is safe to run bitcoin on a certain OS/environment combination, there is no good reason to include these. Better to leave them out to reduce the download size. Sizes from the 0.12 release: ``` 2.4M bitcoin-0.12.0/bin/bench_bitcoin.exe 22M bitcoin-0.12.0/bin/test_bitcoin-qt.exe ``` --- configure.ac | 21 ++++++++++++--------- contrib/gitian-descriptors/gitian-linux.yml | 2 +- contrib/gitian-descriptors/gitian-osx.yml | 2 +- contrib/gitian-descriptors/gitian-win.yml | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/configure.ac b/configure.ac index 81cb894e2..7662c22e3 100644 --- a/configure.ac +++ b/configure.ac @@ -97,6 +97,11 @@ AC_ARG_ENABLE(tests, [use_tests=$enableval], [use_tests=yes]) +AC_ARG_ENABLE(gui-tests, + AS_HELP_STRING([--disable-gui-tests],[do not compile GUI tests (default is to compile if GUI and tests enabled)]), + [use_gui_tests=$enableval], + [use_gui_tests=$use_tests]) + AC_ARG_WITH([comparison-tool], AS_HELP_STRING([--with-comparison-tool],[path to java comparison tool (requires --enable-tests)]), [use_comparison_tool=$withval], @@ -811,8 +816,8 @@ else fi dnl these are only used when qt is enabled +BUILD_TEST_QT="" if test x$bitcoin_enable_qt != xno; then - BUILD_QT=qt dnl enable dbus support AC_MSG_CHECKING([whether to build GUI with support for D-Bus]) if test x$bitcoin_enable_qt_dbus != xno; then @@ -842,9 +847,9 @@ if test x$bitcoin_enable_qt != xno; then fi AC_MSG_CHECKING([whether to build test_bitcoin-qt]) - if test x$use_tests$bitcoin_enable_qt_test = xyesyes; then + if test x$use_gui_tests$bitcoin_enable_qt_test = xyesyes; then AC_MSG_RESULT([yes]) - BUILD_TEST_QT="test" + BUILD_TEST_QT="yes" else AC_MSG_RESULT([no]) fi @@ -853,9 +858,10 @@ fi AC_MSG_CHECKING([whether to build test_bitcoin]) if test x$use_tests = xyes; then AC_MSG_RESULT([yes]) - BUILD_TEST="test" + BUILD_TEST="yes" else AC_MSG_RESULT([no]) + BUILD_TEST="" fi AC_MSG_CHECKING([whether to reduce exports]) @@ -873,9 +879,9 @@ AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) -AM_CONDITIONAL([ENABLE_TESTS],[test x$use_tests = xyes]) +AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) -AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$use_tests$bitcoin_enable_qt_test = xyesyes]) +AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([USE_COMPARISON_TOOL],[test x$use_comparison_tool != xno]) @@ -903,9 +909,6 @@ AC_SUBST(USE_QRCODE) AC_SUBST(BOOST_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(LEVELDB_TARGET_FLAGS) -AC_SUBST(BUILD_TEST) -AC_SUBST(BUILD_QT) -AC_SUBST(BUILD_TEST_QT) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(GMP_LIBS) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index df8c829fb..36d7444fa 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -25,7 +25,7 @@ files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-unknown-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests LDFLAGS=-static-libstdc++" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="date ar ranlib nm strip" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 664ca9dfe..b8fe821be 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -29,7 +29,7 @@ files: script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin11" - CONFIGFLAGS="--enable-reduce-exports GENISOIMAGE=$WRAP_DIR/genisoimage" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 72ff095b9..372235b61 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -28,7 +28,7 @@ files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports" + CONFIGFLAGS="--enable-reduce-exports --disable-gui-tests" FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip" FAKETIME_PROGS="date makensis zip" From e6fa787a624e5679523af44cfb3f45dd960937ad Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 7 Jun 2016 15:45:02 -0400 Subject: [PATCH 12/18] depends: allow for CONFIG_SITE to be used rather than stealing prefix This does not break any existing prefix behavior, only makes new behavior work. For example: CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure --prefix=/ --- depends/config.site.in | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/depends/config.site.in b/depends/config.site.in index 48c062eb9..28c1f45ad 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -1,25 +1,27 @@ +depends_prefix="`dirname ${ac_site_file}`/.." + cross_compiling=maybe host_alias=@HOST@ ac_tool_prefix=${host_alias}- if test -z $with_boost; then - with_boost=$prefix + with_boost=$depends_prefix fi if test -z $with_qt_plugindir; then - with_qt_plugindir=$prefix/plugins + with_qt_plugindir=$depends_prefix/plugins fi if test -z $with_qt_translationdir; then - with_qt_translationdir=$prefix/translations + with_qt_translationdir=$depends_prefix/translations fi if test -z $with_qt_bindir; then - with_qt_bindir=$prefix/native/bin + with_qt_bindir=$depends_prefix/native/bin fi if test -z $with_protoc_bindir; then - with_protoc_bindir=$prefix/native/bin + with_protoc_bindir=$depends_prefix/native/bin fi # Disable comparison utility (#592) #if test -z $with_comparison_tool; then -# with_comparison_tool=$prefix/native/share/BitcoindComparisonTool_jar/BitcoindComparisonTool.jar +# with_comparison_tool=$depends_prefix/native/share/BitcoindComparisonTool_jar/BitcoindComparisonTool.jar #fi @@ -42,31 +44,31 @@ fi if test x@host_os@ = xmingw32; then if test -z $with_qt_incdir; then - with_qt_incdir=$prefix/include + with_qt_incdir=$depends_prefix/include fi if test -z $with_qt_libdir; then - with_qt_libdir=$prefix/lib + with_qt_libdir=$depends_prefix/lib fi fi -PATH=$prefix/native/bin:$PATH +PATH=$depends_prefix/native/bin:$PATH PKG_CONFIG="`which pkg-config` --static" # These two need to remain exported because pkg-config does not see them # otherwise. That means they must be unexported at the end of configure.ac to # avoid ruining the cache. Sigh. -export PKG_CONFIG_LIBDIR=$prefix/lib/pkgconfig -export PKG_CONFIG_PATH=$prefix/share/pkgconfig +export PKG_CONFIG_LIBDIR=$depends_prefix/lib/pkgconfig +export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig -CPPFLAGS="-I$prefix/include/ $CPPFLAGS" -LDFLAGS="-L$prefix/lib $LDFLAGS" +CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS" +LDFLAGS="-L$depends_prefix/lib $LDFLAGS" CC="@CC@" CXX="@CXX@" OBJC="${CC}" OBJCXX="${CXX}" -CCACHE=$prefix/native/bin/ccache +CCACHE=$depends_prefix/native/bin/ccache if test -n "@AR@"; then AR=@AR@ From acfb4cf7c50958cbda268822c546b0d3e0725f1d Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 7 Jun 2016 14:51:51 -0400 Subject: [PATCH 13/18] gitian: use CONFIG_SITE rather than hijacking the prefix --- contrib/gitian-descriptors/gitian-linux.yml | 6 +++--- contrib/gitian-descriptors/gitian-osx.yml | 6 +++--- contrib/gitian-descriptors/gitian-win.yml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 36d7444fa..6b8b5872c 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -73,7 +73,7 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` + CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` @@ -94,11 +94,11 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols - make install-strip + make install-strip DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index b8fe821be..fa86f28c6 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -83,7 +83,7 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` + CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` @@ -105,9 +105,9 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} - make install-strip + make install-strip DESTDIR=${INSTALLPATH} make deploydir mkdir -p unsigned-app-${i} diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 372235b61..8d2dab746 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -100,7 +100,7 @@ script: | # Create the release tarball using (arbitrarily) the first host ./autogen.sh - ./configure --prefix=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'` + CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist SOURCEDIST=`echo bitcoin-*.tar.gz` DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` @@ -124,11 +124,11 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - ./configure --prefix=${BASEPREFIX}/${i} --bindir=${INSTALLPATH}/bin --includedir=${INSTALLPATH}/include --libdir=${INSTALLPATH}/lib --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make deploy - make install-strip + make install-strip DESTDIR=${INSTALLPATH} cp -f bitcoin-*setup*.exe $OUTDIR/ cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ From 2e43dcec21679fcae1a738ead763d6d089d6906d Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 7 Jun 2016 22:09:28 -0400 Subject: [PATCH 14/18] gitian: create debug packages for linux/windows The -debug tarballs/zips contain detached debugging symbols. To use them, place in the same dir as the target binary, and invoke gdb as usual. Also, because the debug symbols add a substantial space requirement, the build dirs are now deleted when they're no longer needed. --- contrib/gitian-descriptors/gitian-linux.yml | 19 ++++++++++++++----- contrib/gitian-descriptors/gitian-win.yml | 18 +++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 6b8b5872c..fefa34032 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -25,9 +25,12 @@ files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-unknown-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests LDFLAGS=-static-libstdc++" + CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="date ar ranlib nm strip" + FAKETIME_PROGS="date ar ranlib nm strip objcopy" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ export QT_RCC_TEST=1 export GZIP="-9n" @@ -94,20 +97,26 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols - make install-strip DESTDIR=${INSTALLPATH} + make install DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME} | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + find ${DISTNAME}/bin -type f -executable -exec objcopy --only-keep-debug {} {}.dbg \; -exec strip -s {} \; -exec objcopy --add-gnu-debuglink={}.dbg {} \; + find ${DISTNAME}/lib -type f -exec objcopy --only-keep-debug {} {}.dbg \; -exec strip -s {} \; -exec objcopy --add-gnu-debuglink={}.dbg {} \; + find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz + find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ + rm -rf distsrc-${i} done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src + mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.tar.gz ${OUTDIR}/${DISTNAME}-linux64-debug.tar.gz + mv ${OUTDIR}/${DISTNAME}-i686-*-debug.tar.gz ${OUTDIR}/${DISTNAME}-linux32-debug.tar.gz mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-linux64.tar.gz mv ${OUTDIR}/${DISTNAME}-i686-*.tar.gz ${OUTDIR}/${DISTNAME}-linux32.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 8d2dab746..e45a270a9 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -29,8 +29,10 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-gui-tests" - FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip" + FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" export QT_RCC_TEST=1 export GZIP="-9n" @@ -124,22 +126,28 @@ script: | mkdir -p ${INSTALLPATH} tar --strip-components=1 -xf ../$SOURCEDIST - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} + CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make deploy - make install-strip DESTDIR=${INSTALLPATH} + make install DESTDIR=${INSTALLPATH} cp -f bitcoin-*setup*.exe $OUTDIR/ cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME} -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip - cd ../.. + find ${DISTNAME}/bin -type f -executable -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; + find ${DISTNAME}/lib -type f -exec ${i}-objcopy --only-keep-debug {} {}.dbg \; -exec ${i}-strip -s {} \; -exec ${i}-objcopy --add-gnu-debuglink={}.dbg {} \; + find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}.zip + find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i}-debug.zip + cd ../../ + rm -rf distsrc-${i} done cd $OUTDIR rename 's/-setup\.exe$/-setup-unsigned.exe/' *-setup.exe find . -name "*-setup-unsigned.exe" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz + mv ${OUTDIR}/${DISTNAME}-x86_64-*-debug.zip ${OUTDIR}/${DISTNAME}-win64-debug.zip + mv ${OUTDIR}/${DISTNAME}-i686-*-debug.zip ${OUTDIR}/${DISTNAME}-win32-debug.zip mv ${OUTDIR}/${DISTNAME}-x86_64-*.zip ${OUTDIR}/${DISTNAME}-win64.zip mv ${OUTDIR}/${DISTNAME}-i686-*.zip ${OUTDIR}/${DISTNAME}-win32.zip From bf5b79510afc2ba93a7fb40898cc48f92566cab3 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 8 Jun 2016 20:21:18 +0200 Subject: [PATCH 15/18] gitian: Add --disable-bench to config flags for windows Forgot to do this in #7776. --- contrib/gitian-descriptors/gitian-win.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index e45a270a9..086fe15e7 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -28,7 +28,7 @@ files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-w64-mingw32 i686-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-gui-tests" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="g++ ar ranlib nm windres strip objcopy" FAKETIME_PROGS="date makensis zip" HOST_CFLAGS="-O2 -g" From 82f8b4899aba19ee3b048ff3fb2a4527b264ea5a Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 25 Dec 2015 13:14:26 +0100 Subject: [PATCH 16/18] [gitian] Default reference_datetime to commit author date --- contrib/gitian-descriptors/gitian-linux.yml | 3 +-- contrib/gitian-descriptors/gitian-osx-signer.yml | 1 - contrib/gitian-descriptors/gitian-osx.yml | 3 +-- contrib/gitian-descriptors/gitian-win-signer.yml | 1 - contrib/gitian-descriptors/gitian-win.yml | 3 +-- 5 files changed, 3 insertions(+), 8 deletions(-) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index fefa34032..c78d27863 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -5,7 +5,7 @@ suites: - "trusty" architectures: - "amd64" -packages: +packages: - "g++-multilib" - "git-core" - "pkg-config" @@ -17,7 +17,6 @@ packages: - "binutils-gold" - "ca-certificates" - "python" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 5b52c492f..b00eb2fb9 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -7,7 +7,6 @@ architectures: packages: - "libc6:i386" - "faketime" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git" "dir": "signature" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index fa86f28c6..953cd439c 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -5,7 +5,7 @@ suites: - "trusty" architectures: - "amd64" -packages: +packages: - "g++" - "git-core" - "pkg-config" @@ -20,7 +20,6 @@ packages: - "libbz2-dev" - "ca-certificates" - "python" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 27c4f01eb..6b53b89b6 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -7,7 +7,6 @@ architectures: packages: - "libssl-dev" - "autoconf" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin-detached-sigs.git" "dir": "signature" diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 086fe15e7..12dbe442b 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -5,7 +5,7 @@ suites: - "trusty" architectures: - "amd64" -packages: +packages: - "g++" - "git-core" - "pkg-config" @@ -20,7 +20,6 @@ packages: - "zip" - "ca-certificates" - "python" -reference_datetime: "2016-01-01 00:00:00" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" From e2d16e0a86053425d740b72580c64417ed567d70 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 5 Apr 2016 13:29:04 +0200 Subject: [PATCH 17/18] [gitian] hardcode datetime for depends --- contrib/gitian-descriptors/gitian-linux.yml | 19 ++++++++++++---- contrib/gitian-descriptors/gitian-osx.yml | 19 ++++++++++++---- contrib/gitian-descriptors/gitian-win.yml | 25 ++++++++++++++++----- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index c78d27863..9a958126c 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -43,29 +43,36 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi - # Create global faketime wrappers + function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done + } - # Create per-host faketime wrappers + function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } + export PATH=${WRAP_DIR}:${PATH} + # Faketime for depends so intermediate results are comparable + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + cd bitcoin BASEPREFIX=`pwd`/depends # Build dependencies for each host @@ -73,6 +80,10 @@ script: | make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done + # Faketime for binaries + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 953cd439c..bc8527039 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -46,29 +46,36 @@ script: | export ZERO_AR_DATE=1 - # Create global faketime wrappers + function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done + } - # Create per-host faketime wrappers + function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } + export PATH=${WRAP_DIR}:${PATH} + # Faketime for depends so intermediate results are comparable + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + cd bitcoin BASEPREFIX=`pwd`/depends @@ -80,6 +87,10 @@ script: | make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done + # Faketime for binaries + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 12dbe442b..1e954f58f 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -45,29 +45,31 @@ script: | mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi - # Create global faketime wrappers + function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done + } - # Create per-host faketime wrappers + function create_per-host_faketime_wrappers { for i in $HOSTS; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } - # Create per-host linker wrapper + function create_per-host_linker_wrapper { # This is only needed for trusty, as the mingw linker leaks a few bytes of # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 for i in $HOSTS; do @@ -83,15 +85,21 @@ script: | echo '#!/bin/bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done + } export PATH=${WRAP_DIR}:${PATH} + # Faketime for depends so intermediate results are comparable + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_linker_wrapper "2000-01-01 12:00:00" + cd bitcoin BASEPREFIX=`pwd`/depends # Build dependencies for each host @@ -99,6 +107,11 @@ script: | make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done + # Faketime for binaries + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_linker_wrapper "${REFERENCE_DATETIME}" + # Create the release tarball using (arbitrarily) the first host ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ From 7d1c2d0b726583680fc067078cfc891b78bd36ae Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 15 Oct 2016 15:55:28 -0500 Subject: [PATCH 18/18] [gitian] Don't call "make check-symbols" Currently it does not recognise any of the C++11 symbols. We can re-enable this when either we update it, or upstream migrates to C++11 and does so. --- contrib/gitian-descriptors/gitian-linux.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 9a958126c..fbaf4c89a 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -110,7 +110,6 @@ script: | CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols make install DESTDIR=${INSTALLPATH} cd installed find . -name "lib*.la" -delete