Browse Source

Add scripts to fetch data bootstrap

pull/303/head
Asher Dawes 4 years ago
parent
commit
fe998bbfe8
  1. 10
      .gitlab-ci.yml
  2. 148
      vcutil/fetch-bootstrap.bat
  3. 296
      vcutil/fetch-bootstrap.sh

10
.gitlab-ci.yml

@ -45,11 +45,14 @@ build:linux:
src/verusd
doc/man/verus-cli/linux/README.txt
zcutil/fetch-params.sh
vcutil/fetch-bootstrap.sh
verus-cli
- mv verus-cli/fetch-params.sh verus-cli/fetch-params
- mv verus-cli/fetch-bootstrap.sh verus-cli/fetch-bootstrap
- chmod +x verus-cli/verus
- chmod +x verus-cli/verusd
- chmod +x verus-cli/fetch-params
- chmod +x verus-cli/fetch-bootstrap
- tar -czvf ${VERUS_CLI_LINUX_AMD64} verus-cli
- sha256sum ${VERUS_CLI_LINUX_AMD64} > ${VERUS_CLI_LINUX_AMD64}.sha256
- git status
@ -85,11 +88,14 @@ build:linux:arm64:
src/verusd
doc/man/verus-cli/linux/README.txt
zcutil/fetch-params.sh
vcutil/fetch-bootstrap.sh
verus-cli
- mv verus-cli/fetch-params.sh verus-cli/fetch-params
- mv verus-cli/fetch-bootstrap.sh verus-cli/fetch-bootstrap
- chmod +x verus-cli/verus
- chmod +x verus-cli/verusd
- chmod +x verus-cli/fetch-params
- chmod +x verus-cli/fetch-bootstrap
- tar -czvf ${VERUS_CLI_ARM64_LINUX} verus-cli
- sha256sum ${VERUS_CLI_ARM64_LINUX} > ${VERUS_CLI_ARM64_LINUX}.sha256
- git status
@ -126,6 +132,7 @@ build:windows:
src/verusd.exe
doc/man/verus-cli/windows/README.txt
zcutil/fetch-params.bat
vcutil/fetch-bootstrap.bat
verus-cli
- if [ "${STRIP_BINARIES}" = "true" ]; then strip --strip-unneeded verus-cli/verusd.exe && strip --strip-unneeded verus-cli/verus.exe; fi
- zip -r ${VERUS_CLI_WINDOWS} verus-cli
@ -161,9 +168,12 @@ build:mac:
src/verusd
doc/man/verus-cli/mac/README.txt
zcutil/fetch-params.sh
vcutil/fetch-bootstrap.sh
verus-cli
- mv verus-cli/fetch-params.sh verus-cli/fetch-params
- mv verus-cli/fetch-bootstrap.sh verus-cli/fetch-bootstrap
- chmod +x verus-cli/fetch-params
- chmod +x verus-cli/fetch-bootstrap
- chmod +x verus-cli/verus
- chmod +x verus-cli/verusd
- tar -czvf ${VERUS_CLI_MACOS} verus-cli

148
vcutil/fetch-bootstrap.bat

@ -0,0 +1,148 @@
@ECHO OFF
TASKLIST /FI "IMAGENAME eq verusd.exe" 2>NUL | find /I /N "verusd.exe">NUL
if "%ERRORLEVEL%"=="0" exit 1
SET PROCESS_NAME=Verus Bootstrap
TASKLIST /V /NH /FI "imagename eq cmd.exe"| FIND /I /C "%PROCESS_NAME%" >NUL
IF %ERRORLEVEL%==0 exit 1
TITLE %PROCESS_NAME%
SETLOCAL EnableDelayedExpansion
SET BOOTSTRAP_URL=https://bootstrap.veruscoin.io
SET TAR_FOUND=0
FOR %%x in (tar.exe) DO IF NOT [%%~$PATH:x]==[] SET TAR_FOUND=1
IF %TAR_FOUND% EQU 1 (
SET BOOTSTRAP_PACKAGE=VRSC-bootstrap.tar.gz
) ELSE (
SET BOOTSTRAP_PACKAGE=VRSC-bootstrap.zip
)
SET BOOTSTRAP_PACKAGE_SIG=!BOOTSTRAP_PACKAGE!.verusid
CALL :MAIN
PAUSE
EXIT 0
:MAIN
CD !Temp!
SET "DOWNLOAD_CMD="
FOR %%x IN (CURL.EXE BITSADMIN.EXE) DO IF NOT [%%~$PATH:x]==[] IF NOT DEFINED DOWNLOAD_CMD SET "DOWNLOAD_CMD=FETCH_%%x"
CALL :SET_INSTALL_DIR
SET "USE_BOOTSTRAP=1"
SET i=0
IF NOT EXIST "!VRSC_DATA_DIR!" (
ECHO No VRSC data directory found, creating directory.
MD "!VRSC_DATA_DIR!"
)
FOR %%F IN (fee_estimates.dat, komodostate, komodostate.ind, peers.dat, db.log, debug.log, signedmasks) DO (
IF EXIST "!VRSC_DATA_DIR!\%%F" (
ECHO Found "!VRSC_DATA_DIR!\%%F"
SET USE_BOOTSTRAP=0
)
)
FOR /D %%D IN (blocks, chainstate, database, notarisations) DO (
IF EXIST "!VRSC_DATA_DIR!\%%D" (
ECHO Found "!VRSC_DATA_DIR!\%%D"
SET USE_BOOTSTRAP=0
)
)
IF /I "!USE_BOOTSTRAP!" EQU "0" (
CHOICE /C:nyq /N /M "Existing blockchain data found. Overwrite? ([y]es/[n]o/[q]uit)"%1
IF !ERRORLEVEL! EQU 3 EXIT 0
SET OVERWRITE_BLOCK_DATA=!ERRORLEVEL!
IF !ERRORLEVEL! EQU 2 (
FOR %%F IN (fee_estimates.dat, komodostate, komodostate.ind, peers.dat, db.log, debug.log, signedmasks) DO (
IF EXIST "!VRSC_DATA_DIR!\%%F" (
ECHO Removing "!VRSC_DATA_DIR!\%%F"
DEL /Q/S "!VRSC_DATA_DIR!\%%F" >NUL
)
)
FOR /D %%D IN (blocks, chainstate, database, notarisations) DO (
IF EXIST "!VRSC_DATA_DIR!\%%D" (
ECHO Removing "!VRSC_DATA_DIR!\%%D"
DEL /Q/S "!VRSC_DATA_DIR!\%%D" >NUL
)
)
CALL :FETCH_BOOTSTRAP
) ELSE (
ECHO Bootstrap not installed
PAUSE
EXIT 0
)
) ELSE (
CALL :FETCH_BOOTSTRAP
)
GOTO :EOF
:SET_INSTALL_DIR
SET VRSC_DATA_DIR=""
SET /P VRSC_DATA_DIR=Enter blockchain data directory or leave blank for default:
IF !VRSC_DATA_DIR! == "" (
SET "VRSC_DATA_DIR=%APPDATA%\Komodo\VRSC"
)
CHOICE /C:nyq /N /M "Install bootstrap in !VRSC_DATA_DIR!? ([y]es/[n]o/[q]uit)"%1
IF !ERRORLEVEL! EQU 3 EXIT 0
IF !ERRORLEVEL! NEQ 2 GOTO SET_INSTALL_DIR
GOTO :EOF
:FETCH_BITSADMIN.EXE
SET "filename=%~1"
SET "URL=%~2"
CALL bitsadmin /transfer "Downloading %filename%" /priority FOREGROUND /download "%URL%/%filename%" "%Temp%\%filename%"
GOTO :EOF
:FETCH_CURL.EXE
SET "filename=%~1"
SET "URL=%~2"
curl -# -L -C - "%URL%/%filename%" -o "%Temp%/%filename%"
GOTO :EOF
:GET_SHA256SUM
SET "file=!%~1!"
SET "sha256sum="
FOR /f "skip=1 tokens=* delims=" %%# IN ('certutil -hashfile !file! SHA256') DO (
IF NOT DEFINED sha256sum (
FOR %%Z IN (%%#) DO SET "sha256sum=!sha256sum!%%Z"
)
)
SET "%~2=!sha256sum!"
GOTO :EOF
:WRITE_BOOTSTRAP_README
(
ECHO !BOOTSTRAP_PACKAGE! needs be extracted directly into this directory. After extration, blocks and chainstate folders should be in this directory.
ECHO !BOOTSTRAP_PACKAGE! can be deleted after extraction.
)>"!VRSC_DATA_DIR!\BOOTSTRAP-README.txt"
GOTO :EOF
:FETCH_BOOTSTRAP
ECHO Fetching VRSC bootstrap
CALL :!DOWNLOAD_CMD! !BOOTSTRAP_PACKAGE! !BOOTSTRAP_URL!
CALL :!DOWNLOAD_CMD! !BOOTSTRAP_PACKAGE_SIG! !BOOTSTRAP_URL!
ECHO Verifying download
SET "filehash="
CALL :GET_SHA256SUM "!Temp!\!BOOTSTRAP_PACKAGE!" filehash
FINDSTR /m "!filehash!" "!Temp!\!BOOTSTRAP_PACKAGE_SIG!" >Nul
IF !ERRORLEVEL! EQU 0 (
ECHO Checksum verified!
IF %TAR_FOUND% EQU 1 (
ECHO Extracting Verus blockchain bootstrap
tar -xf "!Temp!\!BOOTSTRAP_PACKAGE!" --directory "!VRSC_DATA_DIR!"
ECHO Bootstrap successfully installed at "!VRSC_DATA_DIR!"
) ELSE (
MOVE "!Temp!\!BOOTSTRAP_PACKAGE!" "!VRSC_DATA_DIR!"
CALL :WRITE_BOOTSTRAP_README
ECHO tar not found. Opening installation dir for manual bootstrap extraction.
START !VRSC_DATA_DIR!
)
) ELSE (
ECHO "!filehash!"
ECHO Failed to verify bootstrap checksum
)
FOR %%F IN (!BOOTSTRAP_PACKAGE!, !BOOTSTRAP_PACKAGE_SIG!) DO (
IF EXIST "!Temp!\%%F" (
DEL "!Temp!\%%F"
)
)
GOTO :EOF
ENDLOCAL

296
vcutil/fetch-bootstrap.sh

@ -0,0 +1,296 @@
#!/bin/bash
set -eu
# set chain data dir
if [[ -z "${V_CHAIN_DATA_DIR-}" ]]; then
if [[ "$OSTYPE" == "darwin"* ]]; then
V_CHAIN_DATA_DIR="$HOME/Library/Application Support/Komodo/VRSC"
else
V_CHAIN_DATA_DIR="$HOME/.komodo/VRSC"
fi
fi
BOOTSTRAP_URL="https://bootstrap.veruscoin.io"
BOOTSTRAP_ARCHIVE="VRSC-bootstrap.tar.gz"
BOOTSTRAP_ARCHIVE_SIG="$BOOTSTRAP_ARCHIVE.verusid"
SHA256CMD="$(command -v sha256sum || echo shasum)"
SHA256ARGS="$(command -v sha256sum >/dev/null || echo '-a 256')"
WGETCMD="$(command -v wget || echo '')"
IPFSCMD="$(command -v ipfs || echo '')"
CURLCMD="$(command -v curl || echo '')"
PIDOFCMD="$(command -v pidof || echo '')"
PGREPCMD="$(command -v curl || echo '')"
PROCESS_RUNNING=
# fetch methods can be disabled with ZC_DISABLE_SOMETHING=1
ZC_DISABLE_WGET="${ZC_DISABLE_WGET:-}"
ZC_DISABLE_IPFS="${ZC_DISABLE_IPFS:-}"
ZC_DISABLE_CURL="${ZC_DISABLE_CURL:-}"
# overwrite chain data if found
OVERWRITE_BLOCKCHAIN_DATA="${OVERWRITE_BLOCKCHAIN_DATA:-}"
function check_pidof() {
if [ -z "${PIDOFCMD}" ]; then
return 1
fi
local processname="$1"
cat <<EOF
Checking if $processname is running
EOF
pidof "${processname}" >/dev/null
# Check the exit code of the shasum command:
PIDOF_RESULT=$?
if [ $PIDOF_RESULT -eq 0 ]; then
PROCESS_RUNNING=1
fi
}
function check_pgrep() {
if [ -z "${PGREPCMD}" ]; then
return 1
fi
local processname="$1"
cat <<EOF
Checking if $processname is running
EOF
pgrep -x "${processname}" >/dev/null
# Check the exit code of the shasum command:
PGREP_RESULT=$?
if [ $PGREP_RESULT -eq 0 ]; then
PROCESS_RUNNING=1
fi
}
function check_failure() {
cat >&2 <<EOF
Make sure to have either pidof or pgrep installed on your system
EOF
exit 1
}
function fetch_wget() {
if [ -z "$WGETCMD" ] || [ -n "$ZC_DISABLE_WGET" ]; then
return 1
fi
local filename="$1"
local dlname="$2"
local url="$3"
cat <<EOF
Retrieving (wget): ${url}/${filename}
EOF
wget \
--progress=dot:giga \
--output-document="${dlname}" \
--continue \
--retry-connrefused --waitretry=3 --timeout=30 \
"${url}/${filename}"
}
function fetch_ipfs() {
if [ -z "${IPFSCMD}" ] || [ -n "${ZC_DISABLE_IPFS}" ]; then
return 1
fi
local filename="$1"
local dlname="$2"
local cid="$3"
cat <<EOF
Retrieving (ipfs): ${cid}/$filename
EOF
ipfs get --output "${dlname}" "${cid}/${filename}"
}
function fetch_curl() {
if [ -z "${CURLCMD}" ] || [ -n "${ZC_DISABLE_CURL}" ]; then
return 1
fi
local filename="$1"
local dlname="$2"
local url="$3"
cat <<EOF
Retrieving (curl): ${url}/${filename}
EOF
curl \
--output "${dlname}" \
-# -L -C - \
"${url}/${filename}"
}
function fetch_failure() {
cat >&2 <<EOF
Failed to fetch data bootstrap
Make sure one of the following programs installed and make sure you're online:
* ipfs
* wget
* curl
EOF
exit 1
}
function verify_checksum() {
local filename="$1"
local dlname="$2"
local expectedhash="$3"
cat <<EOF
Verifying $filename checksum
EOF
"$SHA256CMD" $SHA256ARGS -c <<EOF
$expectedhash $dlname
EOF
}
# Use flock to prevent parallel execution.
function lock() {
local lockfile=/tmp/fetch_bootstrap.lock
if [[ "$OSTYPE" == "darwin"* ]]; then
if shlock -f ${lockfile} -p $$; then
return 0
else
return 1
fi
else
# create lock file
eval "exec 200>$lockfile"
# acquire the lock
flock -n 200 &&
return 0 ||
return 1
fi
}
function exit_locked_error() {
echo "Only one instance of fetch-bootstrap.sh can be run at a time." >&2
exit 1
}
function overwrite_bootstrap_data() {
for method in pgrep pidof failure; do
if "check_$method" verusd; then
if [ -z "$PROCESS_RUNNING" ]; then
for item in "${vrsc_data[@]}"; do
echo "Removing ${item}"
rm -rf "${item}"
done
fetch_bootstrap
else
echo Verusd is running, close and try again.
fi
break
fi
done
}
function fetch_bootstrap() {
echo Fetching bootstrap
for method in wget curl failure; do
if "fetch_$method" "$BOOTSTRAP_ARCHIVE" "/tmp/$BOOTSTRAP_ARCHIVE" "${BOOTSTRAP_URL}"; then
echo "Download successful!"
break
fi
done
for method in wget curl failure; do
if "fetch_$method" "$BOOTSTRAP_ARCHIVE_SIG" "/tmp/$BOOTSTRAP_ARCHIVE_SIG" "${BOOTSTRAP_URL}"; then
echo "Download successful!"
break
fi
done
expectedhash="$(awk -F'[, \t]*' '/hash/{print substr($3,2,length($3)-2)}' /tmp/$BOOTSTRAP_ARCHIVE_SIG)"
if verify_checksum $BOOTSTRAP_ARCHIVE_SIG /tmp/$BOOTSTRAP_ARCHIVE "$expectedhash"; then
echo Extracting bootstrap
tar -xzf "/tmp/$BOOTSTRAP_ARCHIVE" --directory "${V_CHAIN_DATA_DIR}"
echo Bootstrap successfully installed
rm /tmp/$BOOTSTRAP_ARCHIVE_SIG
rm /tmp/$BOOTSTRAP_ARCHIVE
else
echo "Failed to verify bootstrap checksum!" >&2
rm /tmp/$BOOTSTRAP_ARCHIVE_SIG
rm /tmp/$BOOTSTRAP_ARCHIVE
fi
}
function main() {
lock fetch-bootstrap.sh ||
exit_locked_error
cat <<EOF
This script will install a blockchain data bootstrap
EOF
data_files=("fee_estimates.dat" "komodostate" "komodostate.ind" "peers.dat" "db.log" "debug.log" "signedmasks")
data_dirs=("blocks" "chainstate" "database" "notarisations")
vrsc_data=()
if ! [ -d "${V_CHAIN_DATA_DIR}" ]; then
echo "making dir ${V_CHAIN_DATA_DIR}"
mkdir -p "${V_CHAIN_DATA_DIR}"
ls "${V_CHAIN_DATA_DIR}"
else
for file in "${data_files[@]}"; do
if [ -f "${V_CHAIN_DATA_DIR}/${file}" ]; then
vrsc_data+=("${V_CHAIN_DATA_DIR}"/"${file}")
fi
done
for dir in "${data_dirs[@]}"; do
if [ -d "${V_CHAIN_DATA_DIR}/${dir}" ]; then
vrsc_data+=("${V_CHAIN_DATA_DIR}"/"${dir}")
fi
done
fi
if [ ${#vrsc_data[*]} -lt 1 ]; then
cd "${V_CHAIN_DATA_DIR}"
echo Fetching bootstrap
fetch_bootstrap
else
echo "Found existing VRSC data:"
echo "####################################################################################"
for item in "${vrsc_data[@]}"; do
echo "${item}"
done
echo "####################################################################################"
if [ -n "$OVERWRITE_BLOCKCHAIN_DATA" ]; then
overwrite_bootstrap_data
else
echo "Do you wish to overwrite blockchain data?"
select yn in "Yes" "No"; do
case $yn in
Yes)
overwrite_bootstrap_data
fetch_bootstrap
break
;;
No) echo bootstrap not installed && exit ;;
esac
done
fi
fi
echo Setup complete
}
main
rm -f /tmp/fetch_bootstrap
exit 0
Loading…
Cancel
Save